async def test_create_client_with_api_version(self):
url = self.account_url(self.tables_storage_account_name, "table")
client = TableServiceClient(url, credential=self.credential)
assert client._client._config.version == "2019-02-02"
table = client.get_table_client('tablename')
assert table._client._config.version == "2019-02-02"
client = TableServiceClient(url, credential=self.credential, api_version="2019-07-07")
assert client._client._config.version == "2019-07-07"
table = client.get_table_client('tablename')
assert table._client._config.version == "2019-07-07"
with pytest.raises(ValueError):
TableServiceClient(url, credential=self.credential, api_version="foo")
async def test_set_table_acl_with_signed_identifiers(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
url = self.account_url(tables_cosmos_account_name, "cosmos")
ts = TableServiceClient(url, tables_primary_cosmos_account_key)
table = await self._create_table(ts)
client = ts.get_table_client(table_name=table.table_name)
# Act
identifiers = dict()
identifiers['testid'] = AccessPolicy(
start=datetime.utcnow() - timedelta(minutes=5),
expiry=datetime.utcnow() + timedelta(hours=1),
permission=TableSasPermissions(read=True))
try:
await client.set_table_access_policy(signed_identifiers=identifiers
)
# Assert
acl = await client.get_table_access_policy()
assert acl is not None
assert len(acl) == 1
assert 'testid' in acl
finally:
await ts.delete_table(table.table_name)
if self.is_live:
sleep(SLEEP_DELAY)
async def test_set_table_acl_with_signed_identifiers(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
url = self.account_url(storage_account, "table")
ts = TableServiceClient(url, storage_account_key)
table = await self._create_table(ts)
client = ts.get_table_client(table_name=table.table_name)
# Act
identifiers = dict()
identifiers['testid'] = AccessPolicy(
start=datetime.utcnow() - timedelta(minutes=5),
expiry=datetime.utcnow() + timedelta(hours=1),
permission=TableSasPermissions(read=True))
try:
await client.set_table_access_policy(signed_identifiers=identifiers
)
# Assert
acl = await client.get_table_access_policy()
assert acl is not None
assert len(acl) == 1
assert 'testid' in acl
finally:
await ts.delete_table(table.table_name)
async def test_create_service_with_socket_timeout_async(self):
# Arrange
for service_type in SERVICES.items():
# Act
default_service = service_type[0](self.account_url(
self.tables_storage_account_name, "table"),
credential=self.credential,
table_name='foo')
service = service_type[0](self.account_url(
self.tables_storage_account_name, "table"),
credential=self.credential,
table_name='foo',
connection_timeout=22)
# Assert
self.validate_standard_account_endpoints(
service, self.tables_storage_account_name,
self.tables_primary_storage_account_key)
assert service._client._client._pipeline._transport.connection_config.timeout == 22
assert default_service._client._client._pipeline._transport.connection_config.timeout == 300
# Assert Parent transport is shared with child client
service = TableServiceClient(self.account_url(
self.tables_storage_account_name, "table"),
credential=self.credential,
connection_timeout=22)
assert service._client._client._pipeline._transport.connection_config.timeout == 22
table = service.get_table_client('tablename')
assert table._client._client._pipeline._transport._transport.connection_config.timeout == 22
async def test_set_table_acl_with_signed_identifiers(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
url = self.account_url(storage_account, "table")
if 'cosmos' in url:
pytest.skip("Cosmos endpoint does not support this")
ts = TableServiceClient(url, storage_account_key)
table = await self._create_table(ts)
client = ts.get_table_client(table_name=table.table_name)
# Act
identifiers = dict()
identifiers['testid'] = AccessPolicy(
start=datetime.utcnow() - timedelta(minutes=5),
expiry=datetime.utcnow() + timedelta(hours=1),
permission=TableSasPermissions(read=True))
try:
await client.set_table_access_policy(signed_identifiers=identifiers
)
# Assert
acl = await client.get_table_access_policy()
self.assertIsNotNone(acl)
self.assertEqual(len(acl), 1)
self.assertTrue('testid' in acl)
finally:
# self._delete_table(table)
await ts.delete_table(table.table_name)
async def test_set_table_acl_with_signed_identifiers(
self, tables_storage_account_name,
tables_primary_storage_account_key):
# Arrange
url = self.account_url(tables_storage_account_name, "table")
ts = TableServiceClient(url,
credential=tables_primary_storage_account_key)
table = await self._create_table(ts)
client = ts.get_table_client(table_name=table.table_name)
# Act
start = datetime(2021, 6, 8, 2, 10, 9) - timedelta(minutes=5)
expiry = datetime(2021, 6, 8, 2, 10, 9) + timedelta(hours=1)
identifiers = {
'testid':
TableAccessPolicy(start=start,
expiry=expiry,
permission=TableSasPermissions(read=True))
}
try:
await client.set_table_access_policy(signed_identifiers=identifiers
)
# Assert
acl = await client.get_table_access_policy()
assert acl is not None
assert len(acl) == 1
assert acl.get('testid')
self._assert_policy_datetime(start, acl['testid'].start)
self._assert_policy_datetime(expiry, acl['testid'].expiry)
assert acl['testid'].permission == 'r'
finally:
await ts.delete_table(table.table_name)
async def test_batch_sas_auth(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
token = self.generate_sas(
generate_table_sas,
tables_primary_cosmos_account_key,
self.table_name,
permission=TableSasPermissions(add=True,
read=True,
update=True,
delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1),
)
token = AzureSasCredential(token)
# Act
service = TableServiceClient(
self.account_url(tables_cosmos_account_name, "cosmos"),
credential=token,
)
table = service.get_table_client(self.table_name)
entity = TableEntity()
entity['PartitionKey'] = 'batch_inserts'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
batch = []
transaction_count = 0
for i in range(10):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await table.submit_transaction(batch)
assert transaction_result is not None
total_entities = 0
async for e in table.list_entities():
total_entities += 1
assert total_entities == transaction_count
finally:
await self._tear_down()
async def test_batch_sas_auth(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
token = self.generate_sas(
generate_table_sas,
tables_cosmos_account_name,
tables_primary_cosmos_account_key,
self.table_name,
permission=TableSasPermissions(add=True,
read=True,
update=True,
delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1),
)
token = AzureSasCredential(token)
# Act
service = TableServiceClient(
self.account_url(tables_cosmos_account_name, "cosmos"),
credential=token,
)
table = service.get_table_client(self.table_name)
entity = TableEntity()
entity['PartitionKey'] = 'batch_inserts'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
batch = []
transaction_count = 0
for i in range(10):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await table.submit_transaction(batch)
assert transaction_result is not None
total_entities = 0
async for e in table.list_entities():
total_entities += 1
assert total_entities == transaction_count
finally:
await self._tear_down()
async def test_batch_sas_auth(self, tables_storage_account_name,
tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key)
try:
token = generate_table_sas(
tables_storage_account_name,
tables_primary_storage_account_key,
self.table_name,
permission=TableSasPermissions(add=True,
read=True,
update=True,
delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1),
)
token = AzureSasCredential(token)
# Act
service = TableServiceClient(
self.account_url(tables_storage_account_name, "table"),
credential=token,
)
table = service.get_table_client(self.table_name)
entity = TableEntity()
entity.PartitionKey = 'batch_inserts'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
batch = table.create_batch()
transaction_count = 0
for i in range(10):
entity.RowKey = str(i)
batch.create_entity(entity)
transaction_count += 1
transaction_result = await table.send_batch(batch)
assert transaction_result is not None
total_entities = 0
async for e in table.list_entities():
total_entities += 1
assert total_entities == transaction_count
finally:
await self._tear_down()
async def test_create_table(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
account_url = self.account_url(tables_storage_account_name, "table")
ts = TableServiceClient(credential=tables_primary_storage_account_key, endpoint=account_url)
table_name = self._get_table_reference()
# Act
table = ts.get_table_client(table_name)
created = await table.create_table()
# Assert
assert created.name == table_name
await ts.delete_table(table_name=table_name)
async def test_create_table(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
ts = TableServiceClient(self.account_url(tables_cosmos_account_name,
"cosmos"),
credential=tables_primary_cosmos_account_key)
table_name = self._get_table_reference()
# Act
table = ts.get_table_client(table_name)
created = await table.create_table()
# Assert
assert created.name == table_name
await ts.delete_table(table_name=table_name)
async def test_account_sas(self, tables_storage_account_name,
tables_primary_storage_account_key):
account_url = self.account_url(tables_storage_account_name, "table")
tsc = TableServiceClient(credential=tables_primary_storage_account_key,
endpoint=account_url)
table = await self._create_table(tsc)
try:
entity = {
'PartitionKey': 'test',
'RowKey': 'test1',
'text': 'hello',
}
await table.upsert_entity(entity=entity)
entity['RowKey'] = 'test2'
await table.upsert_entity(entity=entity)
token = self.generate_sas(
generate_account_sas,
tables_primary_storage_account_key,
resource_types=ResourceTypes(object=True),
permission=AccountSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1),
)
token = AzureSasCredential(token)
account_url = self.account_url(tables_storage_account_name,
"table")
service = TableServiceClient(credential=token,
endpoint=account_url)
# Act
sas_table = service.get_table_client(table.table_name)
entities = []
async for e in sas_table.list_entities():
entities.append(e)
# Assert
assert len(entities) == 2
assert entities[0]['text'] == u'hello'
assert entities[1]['text'] == u'hello'
finally:
await tsc.delete_table(table.table_name)
async def test_account_sas(self, resource_group, location, storage_account,
storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
# Arrange
url = self.account_url(storage_account, "table")
if 'cosmos' in url:
pytest.skip("Cosmos Tables does not yet support sas")
tsc = TableServiceClient(url, storage_account_key)
table = await self._create_table(tsc)
try:
entity = {
'PartitionKey': 'test',
'RowKey': 'test1',
'text': 'hello',
}
await table.upsert_entity(entity=entity)
entity['RowKey'] = 'test2'
await table.upsert_entity(entity=entity)
token = generate_account_sas(
storage_account.name,
storage_account_key,
resource_types=ResourceTypes(object=True),
permission=AccountSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1),
)
# Act
service = TableServiceClient(
self.account_url(storage_account, "table"),
credential=token,
)
sas_table = service.get_table_client(table.table_name)
entities = []
async for e in sas_table.list_entities():
entities.append(e)
# Assert
self.assertEqual(len(entities), 2)
self.assertEqual(entities[0].text.value, 'hello')
self.assertEqual(entities[1].text.value, 'hello')
finally:
await self._delete_table(table=table, ts=tsc)
class StorageRetryTest(AzureTestCase, AsyncTableTestCase):
async def _set_up(self,
tables_storage_account_name,
tables_primary_storage_account_key,
url='table',
default_table=True,
**kwargs):
self.table_name = self.get_resource_name('uttable')
self.ts = TableServiceClient(
self.account_url(tables_storage_account_name, url),
credential=tables_primary_storage_account_key,
**kwargs)
self.table = self.ts.get_table_client(self.table_name)
if self.is_live and default_table:
try:
await self.ts.create_table(self.table_name)
except ResourceExistsError:
pass
self.query_tables = []
# --Test Cases --------------------------------------------
@tables_decorator_async
async def test_retry_on_server_error_async(
self, tables_storage_account_name,
tables_primary_storage_account_key):
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
default_table=False)
try:
callback = ResponseCallback(status=201,
new_status=500).override_status
new_table_name = self.get_resource_name('uttable')
# The initial create will return 201, but we overwrite it with 500 and retry.
# The retry will then get a 409 conflict.
with pytest.raises(ResourceExistsError):
await self.ts.create_table(new_table_name,
raw_response_hook=callback)
finally:
await self.ts.delete_table(new_table_name)
await self._tear_down()
@tables_decorator_async
async def test_retry_on_timeout_async(self, tables_storage_account_name,
tables_primary_storage_account_key):
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
default_table=False,
retry_mode=RetryMode.Exponential,
retry_backoff_factor=1)
callback = ResponseCallback(status=200,
new_status=408).override_first_status
try:
# The initial create will return 201, but we overwrite it with 408 and retry.
# The retry will then succeed.
await self.ts.get_service_properties(raw_response_hook=callback)
finally:
await self._tear_down()
@pytest.mark.live_test_only
@tables_decorator_async
async def test_retry_on_socket_timeout_async(
self, tables_storage_account_name,
tables_primary_storage_account_key):
retry_transport = RetryAioHttpTransport(connection_timeout=11,
read_timeout=0.000000000001)
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
retry_mode=RetryMode.Fixed,
retry_backoff_factor=1,
transport=retry_transport,
default_table=False)
with pytest.raises(AzureError) as error:
await self.ts.get_service_properties()
# 3 retries + 1 original == 4
assert retry_transport.count == 4
# This call should succeed on the server side, but fail on the client side due to socket timeout
self.assertTrue(
'Timeout on reading' in str(error.value),
'Expected socket timeout but got different exception.')
@tables_decorator_async
async def test_no_retry_async(self, tables_storage_account_name,
tables_primary_storage_account_key):
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
retry_total=0,
default_table=False)
new_table_name = self.get_resource_name('uttable')
# Force the create call to error with a 500
callback = ResponseCallback(status=201, new_status=500).override_status
try:
with pytest.raises(HttpResponseError) as error:
await self.ts.create_table(new_table_name,
raw_response_hook=callback)
assert error.value.response.status_code == 500
assert error.value.reason == 'Created'
finally:
await self.ts.delete_table(new_table_name)
await self._tear_down()
class AsyncTableTestCase(TableTestCase):
def get_token_credential(self):
if is_live():
return DefaultAzureCredential()
return self.generate_fake_token()
def generate_fake_token(self):
return AsyncFakeTokenCredential()
def _get_table_reference(self, prefix=TEST_TABLE_PREFIX):
table_name = self.get_resource_name(prefix)
return table_name
async def _create_table(self, ts, prefix=TEST_TABLE_PREFIX, table_list=None):
table_name = self._get_table_reference(prefix)
try:
table = await ts.create_table(table_name)
if table_list is not None:
table_list.append(table)
except ResourceExistsError:
table = ts.get_table_client(table_name)
return table
async def _delete_all_tables(self, account_name, key):
client = TableServiceClient(self.account_url(account_name, "cosmos"), credential=key)
async for table in client.list_tables():
await client.delete_table(table.name)
if self.is_live:
self.sleep(10)
async def _tear_down(self):
if is_live():
async for table in self.ts.list_tables():
await self.ts.delete_table(table.name)
self.test_tables = []
await self.ts.close()
async def _create_query_table(self, entity_count):
"""
Creates a table with the specified name and adds entities with the
default set of values. PartitionKey is set to 'MyPartition' and RowKey
is set to a unique counter value starting at 1 (as a string).
"""
table_name = self.get_resource_name("querytable")
table = await self.ts.create_table(table_name)
self.query_tables.append(table_name)
client = self.ts.get_table_client(table_name)
entity = self._create_random_entity_dict()
for i in range(1, entity_count + 1):
entity["RowKey"] = entity["RowKey"] + str(i)
await client.create_entity(entity=entity)
return client
async def _insert_two_opposite_entities(self, pk=None, rk=None):
entity1 = self._create_random_entity_dict()
resp = await self.table.create_entity(entity1)
partition, row = self._create_pk_rk(pk, rk)
properties = {
"PartitionKey": partition + u"1",
"RowKey": row + u"1",
"age": 49,
"sex": u"female",
"married": False,
"deceased": True,
"optional": None,
"ratio": 5.2,
"evenratio": 6.0,
"large": 39999011,
"Birthday": datetime(1993, 4, 1, tzinfo=tzutc()),
"birthday": datetime(1990, 4, 1, tzinfo=tzutc()),
"binary": b"binary-binary",
"other": EntityProperty(40, EdmType.INT32),
"clsid": uuid.UUID("c8da6455-213e-42d9-9b79-3f9149a57833"),
}
await self.table.create_entity(properties)
return entity1, resp
async def _insert_random_entity(self, pk=None, rk=None):
entity = self._create_random_entity_dict(pk, rk)
metadata = await self.table.create_entity(entity=entity)
return entity, metadata["etag"]
async def _set_up(self, account_name, credential, url="table"):
account_url = self.account_url(account_name, url)
self.ts = TableServiceClient(account_url, credential=credential)
self.table_name = self.get_resource_name("uttable")
self.table = self.ts.get_table_client(self.table_name)
if self.is_live:
try:
await self.ts.create_table(table_name=self.table_name)
except ResourceExistsError:
pass
self.query_tables = []
class StorageTableBatchTest(TableTestCase):
async def _set_up(self, storage_account, storage_account_key):
self.ts = TableServiceClient(
self.account_url(storage_account, "table"), storage_account_key)
self.table_name = self.get_resource_name('uttable')
self.table = self.ts.get_table_client(self.table_name)
if self.is_live:
try:
await self.ts.create_table(self.table_name)
except ResourceExistsError:
pass
self.test_tables = []
async def _tear_down(self):
if self.is_live:
try:
await self.ts.delete_table(self.table_name)
except:
pass
for table_name in self.test_tables:
try:
await self.ts.delete_table(table_name)
except:
pass
await self.table.close()
#--Helpers-----------------------------------------------------------------
def _get_table_reference(self, prefix=TEST_TABLE_PREFIX):
table_name = self.get_resource_name(prefix)
self.test_tables.append(table_name)
return self.ts.get_table_client(table_name)
def _create_pk_rk(self, pk, rk):
try:
pk = pk if pk is not None else self.get_resource_name('pk').decode(
'utf-8')
rk = rk if rk is not None else self.get_resource_name('rk').decode(
'utf-8')
except AttributeError:
pk = pk if pk is not None else self.get_resource_name('pk')
rk = rk if rk is not None else self.get_resource_name('rk')
return pk, rk
def _create_random_entity_dict(self, pk=None, rk=None):
"""
Creates a dictionary-based entity with fixed values, using all
of the supported data types.
"""
# partition = pk if pk is not None else self.get_resource_name('pk').decode('utf-8')
# row = rk if rk is not None else self.get_resource_name('rk').decode('utf-8')
partition, row = self._create_pk_rk(pk, rk)
properties = {
'PartitionKey': partition,
'RowKey': row,
'age': 39,
'sex': u'male',
'married': True,
'deceased': False,
'optional': None,
'ratio': 3.1,
'evenratio': 3.0,
'large': 933311100,
'Birthday': datetime(1973, 10, 4, tzinfo=tzutc()),
'birthday': datetime(1970, 10, 4, tzinfo=tzutc()),
'binary': b'binary',
'other': EntityProperty(value=20, type=EdmType.INT32),
'clsid': uuid.UUID('c9da6455-213d-42c9-9a79-3e9149a57833')
}
return TableEntity(**properties)
def _create_updated_entity_dict(self, partition, row):
'''
Creates a dictionary-based entity with fixed values, with a
different set of values than the default entity. It
adds fields, changes field values, changes field types,
and removes fields when compared to the default entity.
'''
return {
'PartitionKey': partition,
'RowKey': row,
'age': u'abc',
'sex': u'female',
'sign': u'aquarius',
'birthday': datetime(1991, 10, 4, tzinfo=tzutc())
}
def _assert_default_entity(self, entity):
'''
Asserts that the entity passed in matches the default entity.
'''
self.assertEqual(entity['age'].value, 39)
self.assertEqual(entity['sex'].value, 'male')
self.assertEqual(entity['married'], True)
self.assertEqual(entity['deceased'], False)
self.assertFalse("optional" in entity)
self.assertEqual(entity['ratio'], 3.1)
self.assertEqual(entity['evenratio'], 3.0)
self.assertEqual(entity['large'].value, 933311100)
self.assertEqual(entity['Birthday'],
datetime(1973, 10, 4, tzinfo=tzutc()))
self.assertEqual(entity['birthday'],
datetime(1970, 10, 4, tzinfo=tzutc()))
self.assertEqual(entity['binary'].value, b'binary')
self.assertIsInstance(entity['other'], EntityProperty)
self.assertEqual(entity['other'].type, EdmType.INT32)
self.assertEqual(entity['other'].value, 20)
self.assertEqual(entity['clsid'],
uuid.UUID('c9da6455-213d-42c9-9a79-3e9149a57833'))
self.assertTrue('_metadata' in entity)
def _assert_updated_entity(self, entity):
'''
Asserts that the entity passed in matches the updated entity.
'''
self.assertEqual(entity.age.value, 'abc')
self.assertEqual(entity.sex.value, 'female')
self.assertFalse(hasattr(entity, "married"))
self.assertFalse(hasattr(entity, "deceased"))
self.assertEqual(entity.sign.value, 'aquarius')
self.assertFalse(hasattr(entity, "optional"))
self.assertFalse(hasattr(entity, "ratio"))
self.assertFalse(hasattr(entity, "evenratio"))
self.assertFalse(hasattr(entity, "large"))
self.assertFalse(hasattr(entity, "Birthday"))
self.assertEqual(entity.birthday, datetime(1991, 10, 4,
tzinfo=tzutc()))
self.assertFalse(hasattr(entity, "other"))
self.assertFalse(hasattr(entity, "clsid"))
self.assertIsNotNone(entity['_metadata']['etag'])
def _assert_valid_batch_transaction(self, transaction, length):
self.assertIsInstance(transaction, BatchTransactionResult)
self.assertEqual(length, len(transaction.entities))
self.assertEqual(length, len(transaction.results))
self.assertEqual(length, len(transaction.requests))
#--Test cases for batch ---------------------------------------------
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_single_insert(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.create_entity(entity)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
sent_entity = transaction_result.get_entity(entity.RowKey)
self.assertIsNotNone(sent_entity)
e = await self.table.get_entity(row_key=entity.RowKey,
partition_key=entity.PartitionKey)
self.assertEqual(e.test, entity.test.value)
self.assertEqual(e.test2.value, entity.test2)
self.assertEqual(e.test3.value, entity.test3)
self.assertEqual(e.test4.value, entity.test4.value)
self.assertEqual(sent_entity['test'], entity.test.value)
self.assertEqual(sent_entity['test2'], entity.test2)
self.assertEqual(sent_entity['test3'], entity.test3)
self.assertEqual(sent_entity['test4'], entity.test4.value)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_single_update(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
resp = await self.table.create_entity(entity)
self.assertIsNotNone(resp)
entity.test3 = 5
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.update_entity(entity, mode=UpdateMode.MERGE)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
self.assertIsNotNone(transaction_result.get_entity(entity.RowKey))
result = await self.table.get_entity(
row_key=entity.RowKey, partition_key=entity.PartitionKey)
self.assertEqual(result.PartitionKey, u'001')
self.assertEqual(result.RowKey, u'batch_insert')
self.assertEqual(result.test3.value, 5)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_update(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_update'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(u'001', u'batch_update')
self.assertEqual(3, entity.test3.value)
entity.test2 = u'value1'
batch = self.table.create_batch()
batch.update_entity(entity)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
self.assertIsNotNone(transaction_result.get_entity(entity.RowKey))
result = await self.table.get_entity('001', 'batch_update')
self.assertEqual('value1', result.test2.value)
self.assertEqual(entity.PartitionKey, u'001')
self.assertEqual(entity.RowKey, u'batch_update')
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_merge(self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_merge'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
self.assertEqual(3, entity.test3)
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_merge'
entity.test2 = u'value1'
batch = self.table.create_batch()
batch.update_entity(entity, mode=UpdateMode.MERGE)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
self.assertIsNotNone(transaction_result.get_entity(entity.RowKey))
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
self.assertEqual(entity.test2, resp_entity.test2.value)
self.assertEqual(1234567890, resp_entity.test4.value)
self.assertEqual(entity.PartitionKey, resp_entity.PartitionKey)
self.assertEqual(entity.RowKey, resp_entity.RowKey)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_update_if_match(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity=entity)
etag = resp['etag']
# Act
sent_entity = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = self.table.create_batch()
batch.update_entity(sent_entity,
etag=etag,
match_condition=MatchConditions.IfNotModified,
mode=UpdateMode.REPLACE)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
self.assertIsNotNone(
transaction_result.get_entity(sent_entity['RowKey']))
entity = await self.table.get_entity(
partition_key=entity['PartitionKey'], row_key=entity['RowKey'])
self._assert_updated_entity(entity)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_update_if_doesnt_match(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity)
self.assertIsNotNone(resp)
# Act
sent_entity1 = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = self.table.create_batch()
batch.update_entity(
sent_entity1,
etag=u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"',
match_condition=MatchConditions.IfNotModified)
with self.assertRaises(BatchErrorException):
await self.table.send_batch(batch)
# Assert
received_entity = await self.table.get_entity(
entity['PartitionKey'], entity['RowKey'])
self._assert_default_entity(received_entity)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_insert_replace(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert_replace'
entity.test = True
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.upsert_entity(entity)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
self.assertIsNotNone(transaction_result.get_entity(entity.RowKey))
entity = await self.table.get_entity('001', 'batch_insert_replace')
self.assertIsNotNone(entity)
self.assertEqual('value', entity.test2.value)
self.assertEqual(1234567890, entity.test4.value)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_insert_merge(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert_merge'
entity.test = True
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.upsert_entity(entity, mode=UpdateMode.MERGE)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
self.assertIsNotNone(transaction_result.get_entity(entity.RowKey))
entity = await self.table.get_entity('001', 'batch_insert_merge')
self.assertIsNotNone(entity)
self.assertEqual('value', entity.test2.value)
self.assertEqual(1234567890, entity.test4.value)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_delete(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_delete'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_delete')
self.assertEqual(3, entity.test3.value)
batch = self.table.create_batch()
batch.delete_entity(partition_key=entity.PartitionKey,
row_key=entity.RowKey)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
self.assertIsNotNone(transaction_result.get_entity(entity.RowKey))
with self.assertRaises(ResourceNotFoundError):
entity = await self.table.get_entity(
partition_key=entity.PartitionKey, row_key=entity.RowKey)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_inserts(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = 'batch_inserts'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
transaction_count = 0
batch = self.table.create_batch()
for i in range(100):
entity.RowKey = str(i)
batch.create_entity(entity)
transaction_count += 1
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
self.assertIsNotNone(transaction_result.get_entity(entity.RowKey))
entities = self.table.query_entities(
"PartitionKey eq 'batch_inserts'")
length = 0
async for e in entities:
length += 1
# Assert
self.assertIsNotNone(entities)
self.assertEqual(100, length)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_all_operations_together(self, resource_group,
location, storage_account,
storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '003'
entity.RowKey = 'batch_all_operations_together-1'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-2'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
await self.table.create_entity(entity)
transaction_count = 0
batch = self.table.create_batch()
entity.RowKey = 'batch_all_operations_together'
batch.create_entity(entity)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-1'
batch.delete_entity(entity.PartitionKey, entity.RowKey)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-2'
entity.test3 = 10
batch.update_entity(entity)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-3'
entity.test3 = 100
batch.update_entity(entity, mode=UpdateMode.MERGE)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-4'
entity.test3 = 10
batch.upsert_entity(entity)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-5'
batch.upsert_entity(entity, mode=UpdateMode.MERGE)
transaction_count += 1
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
self.assertIsNotNone(
transaction_result.get_entity('batch_all_operations_together'))
self.assertIsNotNone(
transaction_result.get_entity(
'batch_all_operations_together-1'))
self.assertIsNotNone(
transaction_result.get_entity(
'batch_all_operations_together-2'))
self.assertIsNotNone(
transaction_result.get_entity(
'batch_all_operations_together-3'))
self.assertIsNotNone(
transaction_result.get_entity(
'batch_all_operations_together-4'))
self.assertIsNotNone(
transaction_result.get_entity(
'batch_all_operations_together-5'))
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
self.assertEqual(5, length)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_all_operations_together_context_manager(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '003'
entity.RowKey = 'batch_all_operations_together-1'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-2'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
await self.table.create_entity(entity)
async with self.table.create_batch() as batch:
entity.RowKey = 'batch_all_operations_together'
batch.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-1'
batch.delete_entity(entity.PartitionKey, entity.RowKey)
entity.RowKey = 'batch_all_operations_together-2'
entity.test3 = 10
batch.update_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
entity.test3 = 100
batch.update_entity(entity, mode=UpdateMode.MERGE)
entity.RowKey = 'batch_all_operations_together-4'
entity.test3 = 10
batch.upsert_entity(entity)
entity.RowKey = 'batch_all_operations_together-5'
batch.upsert_entity(entity, mode=UpdateMode.MERGE)
# Assert
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
self.assertEqual(4, length)
finally:
await self._tear_down()
@pytest.mark.skip(
"Not sure this is how the batching should operate, will consult w/ Anna"
)
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_reuse(self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
table2 = self._get_table_reference('table2')
table2.create_table()
# Act
entity = TableEntity()
entity.PartitionKey = '003'
entity.RowKey = 'batch_all_operations_together-1'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-2'
batch.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
batch.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
batch.create_entity(entity)
await self.table.send_batch(batch)
with self.assertRaises(HttpResponseError):
resp = await table2.send_batch(batch)
# Assert
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
self.assertEqual(5, length)
finally:
await self._tear_down()
@pytest.mark.skip("This does not throw an error, but it should")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_same_row_operations_fail(self, resource_group,
location, storage_account,
storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = self.table.create_batch()
entity = self._create_updated_entity_dict('001',
'batch_negative_1')
batch.update_entity(entity)
entity = self._create_random_entity_dict('001', 'batch_negative_1')
# Assert
with self.assertRaises(HttpResponseError):
batch.update_entity(entity, mode=UpdateMode.MERGE)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_different_partition_operations_fail(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = self.table.create_batch()
entity = self._create_updated_entity_dict('001',
'batch_negative_1')
batch.update_entity(entity)
entity = self._create_random_entity_dict('002', 'batch_negative_1')
# Assert
with self.assertRaises(ValueError):
batch.create_entity(entity)
finally:
await self._tear_down()
@pytest.mark.skip("This does not throw an error, but it should")
@CachedResourceGroupPreparer(name_prefix="tablestest")
@CachedStorageAccountPreparer(name_prefix="tablestest")
async def test_batch_too_many_ops(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
await self._set_up(storage_account, storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
with self.assertRaises(ValueError):
batch = self.table.create_batch()
for i in range(0, 101):
entity = TableEntity()
entity.PartitionKey = 'large'
entity.RowKey = 'item{0}'.format(i)
batch.create_entity(entity)
# Assert
finally:
await self._tear_down()
class TestTableBatchCosmosAsync(AzureRecordedTestCase, AsyncTableTestCase):
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_single_insert(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('create', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
e = await self.table.get_entity(
row_key=entity['RowKey'], partition_key=entity['PartitionKey'])
assert e['test'] == entity['test'].value
assert e['test2'] == entity['test2']
assert e['test3'] == entity['test3']
assert e['test4'] == entity['test4'].value
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_single_update(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
resp = await self.table.create_entity(entity)
assert resp is not None
entity['test3'] = 5
entity['test5'] = datetime.utcnow()
batch = [('update', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
result = await self.table.get_entity(
row_key=entity['RowKey'], partition_key=entity['PartitionKey'])
assert result['PartitionKey'] == u'001'
assert result['RowKey'] == u'batch_insert'
assert result['test3'] == 5
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_update(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_update'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(u'001', u'batch_update')
assert 3 == entity['test3']
entity['test2'] = u'value1'
batch = [('update', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
result = await self.table.get_entity('001', 'batch_update')
assert 'value1' == result['test2']
assert entity['PartitionKey'] == u'001'
assert entity['RowKey'] == u'batch_update'
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_merge(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_merge'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
assert 3 == entity['test3']
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_merge'
entity['test2'] = u'value1'
batch = [('update', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
assert entity['test2'] == resp_entity['test2']
assert 1234567890 == resp_entity['test4']
assert entity['PartitionKey'] == resp_entity['PartitionKey']
assert entity['RowKey'] == resp_entity['RowKey']
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_update_if_match(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity=entity)
etag = resp['etag']
# Act
sent_entity = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = [('update', sent_entity, {
'etag': etag,
'match_condition': MatchConditions.IfNotModified,
'mode': UpdateMode.REPLACE
})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity(
partition_key=entity['PartitionKey'], row_key=entity['RowKey'])
self._assert_updated_entity(entity)
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_update_if_doesnt_match(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity)
assert resp is not None
# Act
sent_entity1 = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = [('update', sent_entity1, {
'etag': u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"',
'match_condition': MatchConditions.IfNotModified
})]
with pytest.raises(TableTransactionError) as error:
await self.table.submit_transaction(batch)
assert error.value.status_code == 412
assert error.value.error_code == TableErrorCode.update_condition_not_satisfied
# Assert
received_entity = await self.table.get_entity(
entity['PartitionKey'], entity['RowKey'])
self._assert_default_entity(received_entity)
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_insert_replace(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert_replace'
entity['test'] = True
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('upsert', entity, {'mode': UpdateMode.REPLACE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity('001', 'batch_insert_replace')
assert entity is not None
assert 'value' == entity['test2']
assert 1234567890 == entity['test4']
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_insert_merge(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert_merge'
entity['test'] = True
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('upsert', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity('001', 'batch_insert_merge')
assert entity is not None
assert 'value' == entity['test2']
assert 1234567890 == entity['test4']
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_delete(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_delete'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_delete')
assert 3 == entity['test3']
batch = [('delete', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' not in transaction_result[0]
with pytest.raises(ResourceNotFoundError):
entity = await self.table.get_entity(
partition_key=entity['PartitionKey'],
row_key=entity['RowKey'])
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_inserts(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = 'batch_inserts'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
transaction_count = 0
batch = []
for i in range(10):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert 'etag' in transaction_result[0]
entities = self.table.query_entities(
"PartitionKey eq 'batch_inserts'")
length = 0
async for e in entities:
length += 1
# Assert
assert entities is not None
assert 10 == length
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_all_operations_together(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '003'
entity['RowKey'] = 'batch_all_operations_together-1'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-2'
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-3'
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-4'
await self.table.create_entity(entity)
transaction_count = 0
batch = []
entity['RowKey'] = 'batch_all_operations_together'
batch.append((TransactionOperation.CREATE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-1'
batch.append((TransactionOperation.DELETE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-2'
entity['test3'] = 10
batch.append((TransactionOperation.UPDATE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-3'
entity['test3'] = 100
batch.append((TransactionOperation.UPDATE, entity.copy(), {
'mode': UpdateMode.REPLACE
}))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-4'
entity['test3'] = 10
batch.append((TransactionOperation.UPSERT, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-5'
batch.append((TransactionOperation.UPSERT, entity.copy(), {
'mode': UpdateMode.REPLACE
}))
transaction_count += 1
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert 'etag' in transaction_result[0]
assert 'etag' not in transaction_result[1]
assert 'etag' in transaction_result[2]
assert 'etag' in transaction_result[3]
assert 'etag' in transaction_result[4]
assert 'etag' in transaction_result[5]
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
assert 5 == length
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_different_partition_operations_fail(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = []
entity = self._create_updated_entity_dict('001',
'batch_negative_1')
batch.append(('update', entity.copy()))
entity = self._create_random_entity_dict('002', 'batch_negative_1')
batch.append(('update', entity.copy(), {
'mode': UpdateMode.REPLACE
}))
# Assert
with pytest.raises(ValueError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_new_non_existent_table(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
tc = self.ts.get_table_client("doesntexist")
batch = [('create', entity)]
with pytest.raises(TableTransactionError):
resp = await tc.submit_transaction(batch)
# Assert
finally:
await self._tear_down()
@pytest.mark.live_test_only
@cosmos_decorator_async
async def test_new_invalid_key(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
invalid_key = tables_primary_cosmos_account_key.named_key.key[
0:-6] + "==" # cut off a bit from the end to invalidate
tables_primary_cosmos_account_key = AzureNamedKeyCredential(
tables_cosmos_account_name, invalid_key)
credential = AzureNamedKeyCredential(
name=tables_cosmos_account_name,
key=tables_primary_cosmos_account_key.named_key.key)
self.ts = TableServiceClient(self.account_url(
tables_cosmos_account_name, "table"),
credential=credential)
self.table_name = self.get_resource_name('uttable')
self.table = self.ts.get_table_client(self.table_name)
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = [('create', entity)]
with pytest.raises(ClientAuthenticationError):
resp = await self.table.submit_transaction(batch)
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_new_delete_nonexistent_entity(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = [('delete', entity)]
with pytest.raises(TableTransactionError):
resp = await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@pytest.mark.live_test_only # Request bodies are very large
@cosmos_decorator_async
async def test_batch_request_too_large(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
batch = []
entity = {
'PartitionKey': 'pk001',
'Foo': os.urandom(1024 * 64),
'Bar': os.urandom(1024 * 64),
'Baz': os.urandom(1024 * 64)
}
for i in range(20):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
with pytest.raises(RequestTooLargeError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_delete_batch_with_bad_kwarg(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
received = await self.table.get_entity(entity["PartitionKey"],
entity["RowKey"])
good_etag = received.metadata["etag"]
received.metadata[
"etag"] = u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"'
batch = [('delete', received, {
"match_condition": MatchConditions.IfNotModified
})]
with pytest.raises(TableTransactionError) as error:
await self.table.submit_transaction(batch)
assert error.value.status_code == 412
assert error.value.error_code == TableErrorCode.update_condition_not_satisfied
received.metadata["etag"] = good_etag
batch = [('delete', received, {
"match_condition": MatchConditions.IfNotModified
})]
resp = await self.table.submit_transaction(batch)
assert resp is not None
finally:
await self._tear_down()
@pytest.mark.live_test_only
@cosmos_decorator_async
async def test_batch_sas_auth(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
token = self.generate_sas(
generate_table_sas,
tables_primary_cosmos_account_key,
self.table_name,
permission=TableSasPermissions(add=True,
read=True,
update=True,
delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1),
)
token = AzureSasCredential(token)
# Act
service = TableServiceClient(
self.account_url(tables_cosmos_account_name, "cosmos"),
credential=token,
)
table = service.get_table_client(self.table_name)
entity = TableEntity()
entity['PartitionKey'] = 'batch_inserts'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
batch = []
transaction_count = 0
for i in range(10):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await table.submit_transaction(batch)
assert transaction_result is not None
total_entities = 0
async for e in table.list_entities():
total_entities += 1
assert total_entities == transaction_count
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_batch_with_specialchar_partitionkey(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
table2_name = self._get_table_reference('table2')
table2 = self.ts.get_table_client(table2_name)
await table2.create_table()
# Act
entity1 = {
'PartitionKey': "A'aaa\"_bbbb2",
'RowKey': '"A\'aaa"_bbbb2',
'test': '"A\'aaa"_bbbb2'
}
await self.table.submit_transaction([("create", entity1)])
get_entity = await self.table.get_entity(
partition_key=entity1['PartitionKey'],
row_key=entity1['RowKey'])
assert get_entity == entity1
await self.table.submit_transaction([("upsert", entity1, {
'mode': 'merge'
})])
get_entity = await self.table.get_entity(
partition_key=entity1['PartitionKey'],
row_key=entity1['RowKey'])
assert get_entity == entity1
await self.table.submit_transaction([("upsert", entity1, {
'mode': 'replace'
})])
get_entity = await self.table.get_entity(
partition_key=entity1['PartitionKey'],
row_key=entity1['RowKey'])
assert get_entity == entity1
await self.table.submit_transaction([("update", entity1, {
'mode': 'merge'
})])
get_entity = await self.table.get_entity(
partition_key=entity1['PartitionKey'],
row_key=entity1['RowKey'])
assert get_entity == entity1
await self.table.submit_transaction([("update", entity1, {
'mode': 'replace'
})])
get_entity = await self.table.get_entity(
partition_key=entity1['PartitionKey'],
row_key=entity1['RowKey'])
assert get_entity == entity1
entity_results = self.table.list_entities()
async for entity in entity_results:
assert entity == entity1
get_entity = await self.table.get_entity(
partition_key=entity['PartitionKey'],
row_key=entity['RowKey'])
assert get_entity == entity1
await self.table.submit_transaction([("delete", entity1)])
finally:
await self._tear_down()
@cosmos_decorator_async
@recorded_by_proxy_async
async def test_async_batch_inserts(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# this can be reverted to set_bodiless_matcher() after tests are re-recorded and don't contain these headers
set_custom_default_matcher(
compare_bodies=False,
excluded_headers=
"Authorization,Content-Length,x-ms-client-request-id,x-ms-request-id"
)
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
transaction_count = 10
async def generate_entities(count):
for i in range(count):
yield ("upsert", {
'PartitionKey': 'async_inserts',
'RowKey': str(i)
})
batch = generate_entities(transaction_count)
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert 'etag' in transaction_result[0]
entities = self.table.query_entities(
"PartitionKey eq 'async_inserts'")
entities = [e async for e in entities]
# Assert
assert len(entities) == transaction_count
finally:
await self._tear_down()
class StorageTableBatchTest(AzureTestCase, AsyncTableTestCase):
@cosmos_decorator_async
async def test_batch_single_insert(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('create', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
e = await self.table.get_entity(
row_key=entity['RowKey'], partition_key=entity['PartitionKey'])
assert e['test'] == entity['test'].value
assert e['test2'] == entity['test2']
assert e['test3'] == entity['test3']
assert e['test4'] == entity['test4'].value
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_single_update(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
resp = await self.table.create_entity(entity)
assert resp is not None
entity['test3'] = 5
entity['test5'] = datetime.utcnow()
batch = [('update', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
result = await self.table.get_entity(
row_key=entity['RowKey'], partition_key=entity['PartitionKey'])
assert result['PartitionKey'] == u'001'
assert result['RowKey'] == u'batch_insert'
assert result['test3'] == 5
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_update(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_update'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(u'001', u'batch_update')
assert 3 == entity['test3']
entity['test2'] = u'value1'
batch = [('update', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
result = await self.table.get_entity('001', 'batch_update')
assert 'value1' == result['test2']
assert entity['PartitionKey'] == u'001'
assert entity['RowKey'] == u'batch_update'
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_merge(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_merge'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
assert 3 == entity['test3']
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_merge'
entity['test2'] = u'value1'
batch = [('update', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
assert entity['test2'] == resp_entity['test2']
assert 1234567890 == resp_entity['test4']
assert entity['PartitionKey'] == resp_entity['PartitionKey']
assert entity['RowKey'] == resp_entity['RowKey']
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_update_if_match(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity=entity)
etag = resp['etag']
# Act
sent_entity = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = [('update', sent_entity, {
'etag': etag,
'match_condition': MatchConditions.IfNotModified,
'mode': UpdateMode.REPLACE
})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity(
partition_key=entity['PartitionKey'], row_key=entity['RowKey'])
self._assert_updated_entity(entity)
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_update_if_doesnt_match(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity)
assert resp is not None
# Act
sent_entity1 = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = [('update', sent_entity1, {
'etag': u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"',
'match_condition': MatchConditions.IfNotModified
})]
with pytest.raises(HttpResponseError):
await self.table.submit_transaction(batch)
# Assert
received_entity = await self.table.get_entity(
entity['PartitionKey'], entity['RowKey'])
self._assert_default_entity(received_entity)
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_insert_replace(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert_replace'
entity['test'] = True
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('upsert', entity, {'mode': UpdateMode.REPLACE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity('001', 'batch_insert_replace')
assert entity is not None
assert 'value' == entity['test2']
assert 1234567890 == entity['test4']
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_insert_merge(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert_merge'
entity['test'] = True
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('upsert', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity('001', 'batch_insert_merge')
assert entity is not None
assert 'value' == entity['test2']
assert 1234567890 == entity['test4']
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_delete(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_delete'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_delete')
assert 3 == entity['test3']
batch = [('delete', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' not in transaction_result[0]
with pytest.raises(ResourceNotFoundError):
entity = await self.table.get_entity(
partition_key=entity['PartitionKey'],
row_key=entity['RowKey'])
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_inserts(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = 'batch_inserts'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
transaction_count = 0
batch = []
for i in range(10):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert 'etag' in transaction_result[0]
entities = self.table.query_entities(
"PartitionKey eq 'batch_inserts'")
length = 0
async for e in entities:
length += 1
# Assert
assert entities is not None
assert 10 == length
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_all_operations_together(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '003'
entity['RowKey'] = 'batch_all_operations_together-1'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-2'
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-3'
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-4'
await self.table.create_entity(entity)
transaction_count = 0
batch = []
entity['RowKey'] = 'batch_all_operations_together'
batch.append((TransactionOperation.CREATE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-1'
batch.append((TransactionOperation.DELETE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-2'
entity['test3'] = 10
batch.append((TransactionOperation.UPDATE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-3'
entity['test3'] = 100
batch.append((TransactionOperation.UPDATE, entity.copy(), {
'mode': UpdateMode.REPLACE
}))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-4'
entity['test3'] = 10
batch.append((TransactionOperation.UPSERT, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-5'
batch.append((TransactionOperation.UPSERT, entity.copy(), {
'mode': UpdateMode.REPLACE
}))
transaction_count += 1
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert 'etag' in transaction_result[0]
assert 'etag' not in transaction_result[1]
assert 'etag' in transaction_result[2]
assert 'etag' in transaction_result[3]
assert 'etag' in transaction_result[4]
assert 'etag' in transaction_result[5]
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
assert 5 == length
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_batch_different_partition_operations_fail(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = []
entity = self._create_updated_entity_dict('001',
'batch_negative_1')
batch.append(('update', entity.copy()))
entity = self._create_random_entity_dict('002', 'batch_negative_1')
batch.append(('update', entity.copy(), {
'mode': UpdateMode.REPLACE
}))
# Assert
with pytest.raises(ValueError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@cosmos_decorator_async
async def test_new_non_existent_table(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
tc = self.ts.get_table_client("doesntexist")
batch = [('create', entity)]
with pytest.raises(TableTransactionError):
resp = await tc.submit_transaction(batch)
# Assert
finally:
await self._tear_down()
@pytest.mark.live_test_only
@cosmos_decorator_async
async def test_new_invalid_key(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
invalid_key = tables_primary_cosmos_account_key[
0:-6] + "==" # cut off a bit from the end to invalidate
key_list = list(tables_primary_cosmos_account_key)
key_list[-6:] = list("0000==")
invalid_key = ''.join(key_list)
self.ts = TableServiceClient(
self.account_url(tables_cosmos_account_name, "table"), invalid_key)
self.table_name = self.get_resource_name('uttable')
self.table = self.ts.get_table_client(self.table_name)
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = [('create', entity)]
with pytest.raises(ClientAuthenticationError):
resp = await self.table.submit_transaction(batch)
@cosmos_decorator_async
async def test_new_delete_nonexistent_entity(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = [('delete', entity)]
with pytest.raises(TableTransactionError):
resp = await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@pytest.mark.live_test_only # Request bodies are very large
@cosmos_decorator_async
async def test_batch_request_too_large(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
batch = []
entity = {
'PartitionKey': 'pk001',
'Foo': os.urandom(1024 * 64),
'Bar': os.urandom(1024 * 64),
'Baz': os.urandom(1024 * 64)
}
for i in range(20):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
with pytest.raises(RequestTooLargeError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@cosmos_decorator_async
async def test_delete_batch_with_bad_kwarg(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
received = await self.table.get_entity(entity["PartitionKey"],
entity["RowKey"])
good_etag = received.metadata["etag"]
received.metadata[
"etag"] = u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"'
batch = [('delete', received, {
"match_condition": MatchConditions.IfNotModified
})]
with pytest.raises(TableTransactionError):
await self.table.submit_transaction(batch)
received.metadata["etag"] = good_etag
batch = [('delete', received, {
"match_condition": MatchConditions.IfNotModified
})]
resp = await self.table.submit_transaction(batch)
assert resp is not None
finally:
await self._tear_down()
@pytest.mark.live_test_only
@cosmos_decorator_async
async def test_batch_sas_auth(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key,
url="cosmos")
try:
token = self.generate_sas(
generate_table_sas,
tables_cosmos_account_name,
tables_primary_cosmos_account_key,
self.table_name,
permission=TableSasPermissions(add=True,
read=True,
update=True,
delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1),
)
token = AzureSasCredential(token)
# Act
service = TableServiceClient(
self.account_url(tables_cosmos_account_name, "cosmos"),
credential=token,
)
table = service.get_table_client(self.table_name)
entity = TableEntity()
entity['PartitionKey'] = 'batch_inserts'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
batch = []
transaction_count = 0
for i in range(10):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await table.submit_transaction(batch)
assert transaction_result is not None
total_entities = 0
async for e in table.list_entities():
total_entities += 1
assert total_entities == transaction_count
finally:
await self._tear_down()
class StorageTableBatchTest(AzureTestCase, AsyncTableTestCase):
async def _set_up(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
self.ts = TableServiceClient(
self.account_url(tables_cosmos_account_name, "cosmos"),
tables_primary_cosmos_account_key)
self.table_name = self.get_resource_name('uttable')
self.table = self.ts.get_table_client(self.table_name)
if self.is_live:
try:
await self.ts.create_table(self.table_name)
except ResourceExistsError:
pass
self.test_tables = [self.table_name]
async def _tear_down(self):
if self.is_live:
try:
await self.ts.delete_table(self.table_name)
except:
pass
for table_name in self.test_tables:
try:
await self.ts.delete_table(table_name)
except:
pass
await self.table.close()
self.test_tables = []
self.sleep(SLEEP_DELAY)
#--Helpers-----------------------------------------------------------------
def _get_table_reference(self, prefix=TEST_TABLE_PREFIX):
table_name = self.get_resource_name(prefix)
self.test_tables.append(table_name)
return self.ts.get_table_client(table_name)
def _create_pk_rk(self, pk, rk):
try:
pk = pk if pk is not None else self.get_resource_name('pk').decode(
'utf-8')
rk = rk if rk is not None else self.get_resource_name('rk').decode(
'utf-8')
except AttributeError:
pk = pk if pk is not None else self.get_resource_name('pk')
rk = rk if rk is not None else self.get_resource_name('rk')
return pk, rk
def _create_random_entity_dict(self, pk=None, rk=None):
"""
Creates a dictionary-based entity with fixed values, using all
of the supported data types.
"""
# partition = pk if pk is not None else self.get_resource_name('pk').decode('utf-8')
# row = rk if rk is not None else self.get_resource_name('rk').decode('utf-8')
partition, row = self._create_pk_rk(pk, rk)
properties = {
'PartitionKey': partition,
'RowKey': row,
'age': 39,
'sex': u'male',
'married': True,
'deceased': False,
'optional': None,
'ratio': 3.1,
'evenratio': 3.0,
'large': 933311100,
'Birthday': datetime(1973, 10, 4, tzinfo=tzutc()),
'birthday': datetime(1970, 10, 4, tzinfo=tzutc()),
'binary': b'binary',
'other': EntityProperty(value=20, type=EdmType.INT32),
'clsid': uuid.UUID('c9da6455-213d-42c9-9a79-3e9149a57833')
}
return TableEntity(**properties)
def _create_updated_entity_dict(self, partition, row):
'''
Creates a dictionary-based entity with fixed values, with a
different set of values than the default entity. It
adds fields, changes field values, changes field types,
and removes fields when compared to the default entity.
'''
return {
'PartitionKey': partition,
'RowKey': row,
'age': u'abc',
'sex': u'female',
'sign': u'aquarius',
'birthday': datetime(1991, 10, 4, tzinfo=tzutc())
}
def _assert_default_entity(self, entity):
'''
Asserts that the entity passed in matches the default entity.
'''
assert entity['age'] == 39
assert entity['sex'] == 'male'
assert entity['married'] == True
assert entity['deceased'] == False
assert not "optional" in entity
assert entity['ratio'] == 3.1
assert entity['evenratio'] == 3.0
assert entity['large'] == 933311100
assert entity['Birthday'] == datetime(1973, 10, 4, tzinfo=tzutc())
assert entity['birthday'] == datetime(1970, 10, 4, tzinfo=tzutc())
assert entity['binary'].value == b'binary'
assert entity['other'] == 20
assert entity['clsid'] == uuid.UUID(
'c9da6455-213d-42c9-9a79-3e9149a57833')
assert '_metadata' in entity
def _assert_updated_entity(self, entity):
'''
Asserts that the entity passed in matches the updated entity.
'''
assert entity.age == 'abc'
assert entity.sex == 'female'
assert not hasattr(entity, "married")
assert not hasattr(entity, "deceased")
assert entity.sign == 'aquarius'
assert not hasattr(entity, "optional")
assert not hasattr(entity, "ratio")
assert not hasattr(entity, "evenratio")
assert not hasattr(entity, "large")
assert not hasattr(entity, "Birthday")
assert entity.birthday, datetime(1991, 10, 4, tzinfo=tzutc())
assert not hasattr(entity, "other")
assert not hasattr(entity, "clsid")
assert entity['_metadata']['etag'] is not None
def _assert_valid_batch_transaction(self, transaction, length):
assert length == len(transaction)
#--Test cases for batch ---------------------------------------------
@CosmosPreparer()
async def test_batch_single_insert(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = [('create', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
e = await self.table.get_entity(row_key=entity.RowKey,
partition_key=entity.PartitionKey)
assert e.test == entity.test.value
assert e.test2 == entity.test2
assert e.test3 == entity.test3
assert e.test4 == entity.test4.value
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_single_update(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
resp = await self.table.create_entity(entity)
assert resp is not None
entity.test3 = 5
entity.test5 = datetime.utcnow()
batch = [('update', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
result = await self.table.get_entity(
row_key=entity.RowKey, partition_key=entity.PartitionKey)
assert result.PartitionKey == u'001'
assert result.RowKey == u'batch_insert'
assert result.test3 == 5
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_update(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_update'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(u'001', u'batch_update')
assert 3 == entity.test3
entity.test2 = u'value1'
batch = [('update', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
result = await self.table.get_entity('001', 'batch_update')
assert 'value1' == result.test2
assert entity.PartitionKey == u'001'
assert entity.RowKey == u'batch_update'
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_merge(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_merge'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
assert 3 == entity.test3
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_merge'
entity.test2 = u'value1'
batch = [('update', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
assert entity.test2 == resp_entity.test2
assert 1234567890 == resp_entity.test4
assert entity.PartitionKey == resp_entity.PartitionKey
assert entity.RowKey == resp_entity.RowKey
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_update_if_match(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity=entity)
etag = resp['etag']
# Act
sent_entity = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = [('update', sent_entity, {
'etag': etag,
'match_condition': MatchConditions.IfNotModified,
'mode': UpdateMode.REPLACE
})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity(
partition_key=entity['PartitionKey'], row_key=entity['RowKey'])
self._assert_updated_entity(entity)
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_update_if_doesnt_match(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity)
assert resp is not None
# Act
sent_entity1 = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = [('update', sent_entity1, {
'etag': u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"',
'match_condition': MatchConditions.IfNotModified
})]
with pytest.raises(HttpResponseError):
await self.table.submit_transaction(batch)
# Assert
received_entity = await self.table.get_entity(
entity['PartitionKey'], entity['RowKey'])
self._assert_default_entity(received_entity)
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_insert_replace(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert_replace'
entity.test = True
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = [('upsert', entity, {'mode': UpdateMode.REPLACE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity('001', 'batch_insert_replace')
assert entity is not None
assert 'value' == entity.test2
assert 1234567890 == entity.test4
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_insert_merge(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert_merge'
entity.test = True
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = [('upsert', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity('001', 'batch_insert_merge')
assert entity is not None
assert 'value' == entity.test2
assert 1234567890 == entity.test4
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_delete(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_delete'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_delete')
assert 3 == entity.test3
batch = [('delete', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' not in transaction_result[0]
with pytest.raises(ResourceNotFoundError):
entity = await self.table.get_entity(
partition_key=entity.PartitionKey, row_key=entity.RowKey)
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_inserts(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = 'batch_inserts'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
transaction_count = 0
batch = []
for i in range(10):
entity.RowKey = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert 'etag' in transaction_result[0]
entities = self.table.query_entities(
"PartitionKey eq 'batch_inserts'")
length = 0
async for e in entities:
length += 1
# Assert
assert entities is not None
assert 10 == length
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_all_operations_together(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '003'
entity.RowKey = 'batch_all_operations_together-1'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-2'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
await self.table.create_entity(entity)
transaction_count = 0
batch = []
entity.RowKey = 'batch_all_operations_together'
batch.append((TransactionOperation.CREATE, entity.copy()))
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-1'
batch.append((TransactionOperation.DELETE, entity.copy()))
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-2'
entity.test3 = 10
batch.append((TransactionOperation.UPDATE, entity.copy()))
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-3'
entity.test3 = 100
batch.append((TransactionOperation.UPDATE, entity.copy(), {
'mode': UpdateMode.REPLACE
}))
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-4'
entity.test3 = 10
batch.append((TransactionOperation.UPSERT, entity.copy()))
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-5'
batch.append((TransactionOperation.UPSERT, entity.copy(), {
'mode': UpdateMode.REPLACE
}))
transaction_count += 1
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert 'etag' in transaction_result[0]
assert 'etag' not in transaction_result[1]
assert 'etag' in transaction_result[2]
assert 'etag' in transaction_result[3]
assert 'etag' in transaction_result[4]
assert 'etag' in transaction_result[5]
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
assert 5 == length
finally:
await self._tear_down()
@CosmosPreparer()
async def test_batch_different_partition_operations_fail(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = []
entity = self._create_updated_entity_dict('001',
'batch_negative_1')
batch.append(('update', entity.copy()))
entity = self._create_random_entity_dict('002', 'batch_negative_1')
batch.append(('update', entity.copy(), {
'mode': UpdateMode.REPLACE
}))
# Assert
with pytest.raises(ValueError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@CosmosPreparer()
async def test_new_non_existent_table(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
tc = self.ts.get_table_client("doesntexist")
batch = [('create', entity)]
with pytest.raises(TableTransactionError):
resp = await tc.submit_transaction(batch)
# Assert
finally:
await self._tear_down()
@pytest.mark.live_test_only
@CosmosPreparer()
async def test_new_invalid_key(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
invalid_key = tables_primary_cosmos_account_key[
0:-6] + "==" # cut off a bit from the end to invalidate
key_list = list(tables_primary_cosmos_account_key)
key_list[-6:] = list("0000==")
invalid_key = ''.join(key_list)
self.ts = TableServiceClient(
self.account_url(tables_cosmos_account_name, "table"), invalid_key)
self.table_name = self.get_resource_name('uttable')
self.table = self.ts.get_table_client(self.table_name)
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = [('create', entity)]
with pytest.raises(ClientAuthenticationError):
resp = await self.table.submit_transaction(batch)
@CosmosPreparer()
async def test_new_delete_nonexistent_entity(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = [('delete', entity)]
with pytest.raises(TableTransactionError):
resp = await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@pytest.mark.live_test_only # Request bodies are very large
@CosmosPreparer()
async def test_batch_request_too_large(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
batch = []
entity = {
'PartitionKey': 'pk001',
'Foo': os.urandom(1024 * 64),
'Bar': os.urandom(1024 * 64),
'Baz': os.urandom(1024 * 64)
}
for i in range(20):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
with pytest.raises(RequestTooLargeError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
class StorageTableBatchTest(AzureTestCase, AsyncTableTestCase):
async def _set_up(self, tables_storage_account_name, tables_primary_storage_account_key):
self.ts = TableServiceClient(self.account_url(tables_storage_account_name, "table"), tables_primary_storage_account_key)
self.table_name = self.get_resource_name('uttable')
self.table = self.ts.get_table_client(self.table_name)
if self.is_live:
try:
await self.ts.create_table(self.table_name)
except ResourceExistsError:
pass
self.test_tables = []
async def _tear_down(self):
if self.is_live:
try:
await self.ts.delete_table(self.table_name)
except:
pass
for table_name in self.test_tables:
try:
await self.ts.delete_table(table_name)
except:
pass
await self.table.close()
#--Helpers-----------------------------------------------------------------
def _get_table_reference(self, prefix=TEST_TABLE_PREFIX):
table_name = self.get_resource_name(prefix)
self.test_tables.append(table_name)
return self.ts.get_table_client(table_name)
def _create_pk_rk(self, pk, rk):
try:
pk = pk if pk is not None else self.get_resource_name('pk').decode('utf-8')
rk = rk if rk is not None else self.get_resource_name('rk').decode('utf-8')
except AttributeError:
pk = pk if pk is not None else self.get_resource_name('pk')
rk = rk if rk is not None else self.get_resource_name('rk')
return pk, rk
def _create_random_entity_dict(self, pk=None, rk=None):
"""
Creates a dictionary-based entity with fixed values, using all
of the supported data types.
"""
# partition = pk if pk is not None else self.get_resource_name('pk').decode('utf-8')
# row = rk if rk is not None else self.get_resource_name('rk').decode('utf-8')
partition, row = self._create_pk_rk(pk, rk)
properties = {
'PartitionKey': partition,
'RowKey': row,
'age': 39,
'sex': u'male',
'married': True,
'deceased': False,
'optional': None,
'ratio': 3.1,
'evenratio': 3.0,
'large': 933311100,
'Birthday': datetime(1973, 10, 4, tzinfo=tzutc()),
'birthday': datetime(1970, 10, 4, tzinfo=tzutc()),
'binary': b'binary',
'other': EntityProperty(20, EdmType.INT32),
'clsid': uuid.UUID('c9da6455-213d-42c9-9a79-3e9149a57833')
}
return TableEntity(**properties)
def _create_updated_entity_dict(self, partition, row):
'''
Creates a dictionary-based entity with fixed values, with a
different set of values than the default entity. It
adds fields, changes field values, changes field types,
and removes fields when compared to the default entity.
'''
return {
'PartitionKey': partition,
'RowKey': row,
'age': u'abc',
'sex': u'female',
'sign': u'aquarius',
'birthday': datetime(1991, 10, 4, tzinfo=tzutc())
}
def _assert_default_entity(self, entity):
'''
Asserts that the entity passed in matches the default entity.
'''
assert entity['age'] == 39
assert entity['sex'] == 'male'
assert entity['married'] == True
assert entity['deceased'] == False
assert not "optional" in entity
assert entity['ratio'] == 3.1
assert entity['evenratio'] == 3.0
assert entity['large'] == 933311100
assert entity['Birthday'] == datetime(1973, 10, 4, tzinfo=tzutc())
assert entity['birthday'] == datetime(1970, 10, 4, tzinfo=tzutc())
assert entity['binary'].value == b'binary'
assert entity['other'] == 20
assert entity['clsid'] == uuid.UUID('c9da6455-213d-42c9-9a79-3e9149a57833')
assert entity.metadata['etag']
assert entity.metadata['timestamp']
def _assert_updated_entity(self, entity):
'''
Asserts that the entity passed in matches the updated entity.
'''
assert entity['age'] == 'abc'
assert entity['sex'] == 'female'
assert not "married" in entity
assert not "deceased" in entity
assert entity['sign'] == 'aquarius'
assert not "optional" in entity
assert not "ratio" in entity
assert not "evenratio" in entity
assert not "large" in entity
assert not "Birthday" in entity
assert entity['birthday'] == datetime(1991, 10, 4, tzinfo=tzutc())
assert not "other" in entity
assert not "clsid" in entity
assert entity.metadata['etag']
assert entity.metadata['timestamp']
def _assert_valid_batch_transaction(self, transaction, length):
assert length == len(transaction)
#--Test cases for batch ---------------------------------------------
@tables_decorator_async
async def test_batch_single_insert(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('create', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
e = await self.table.get_entity(row_key=entity['RowKey'], partition_key=entity['PartitionKey'])
assert e['test'] == entity['test'].value
assert e['test2'] == entity['test2']
assert e['test3'] == entity['test3']
assert e['test4'] == entity['test4'].value
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_single_update(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
resp = await self.table.create_entity(entity)
assert resp is not None
entity['test3'] = 5
entity['test5'] = datetime.utcnow()
batch = [('update', entity, {'mode':UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
result = await self.table.get_entity(row_key=entity['RowKey'], partition_key=entity['PartitionKey'])
assert result['PartitionKey'] == u'001'
assert result['RowKey'] == u'batch_insert'
assert result['test3'] == 5
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_update(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_update'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(u'001', u'batch_update')
assert 3 == entity['test3']
entity['test2'] = u'value1'
batch = [('update', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
result = await self.table.get_entity('001', 'batch_update')
assert 'value1' == result['test2']
assert entity['PartitionKey'] == u'001'
assert entity['RowKey'] == u'batch_update'
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_merge(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_merge'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
resp_entity = await self.table.get_entity(partition_key=u'001', row_key=u'batch_merge')
assert 3 == entity['test3']
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_merge'
entity['test2'] = u'value1'
batch = [('update', entity, {'mode': UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
resp_entity = await self.table.get_entity(partition_key=u'001', row_key=u'batch_merge')
assert entity['test2'] == resp_entity['test2']
assert 1234567890 == resp_entity['test4']
assert entity['PartitionKey'] == resp_entity['PartitionKey']
assert entity['RowKey'] == resp_entity['RowKey']
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_update_if_match(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity=entity)
etag = resp['etag']
# Act
sent_entity = self._create_updated_entity_dict(entity['PartitionKey'], entity['RowKey'])
batch = [(
'update',
sent_entity,
{'etag': etag, 'match_condition':MatchConditions.IfNotModified, 'mode':UpdateMode.REPLACE}
)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert transaction_result[0]['etag'] != etag
entity = await self.table.get_entity(partition_key=entity['PartitionKey'], row_key=entity['RowKey'])
self._assert_updated_entity(entity)
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_update_if_doesnt_match(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity)
assert resp is not None
# Act
sent_entity1 = self._create_updated_entity_dict(entity['PartitionKey'], entity['RowKey'])
batch = [(
'update',
sent_entity1,
{'etag': u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"', 'match_condition':MatchConditions.IfNotModified}
)]
with pytest.raises(TableTransactionError) as error:
await self.table.submit_transaction(batch)
# Assert
received_entity = await self.table.get_entity(entity['PartitionKey'], entity['RowKey'])
self._assert_default_entity(received_entity)
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_insert_replace(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert_replace'
entity['test'] = True
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('upsert', entity, {'mode': UpdateMode.REPLACE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity('001', 'batch_insert_replace')
assert entity is not None
assert 'value' == entity['test2']
assert 1234567890 == entity['test4']
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_insert_merge(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '001'
entity['RowKey'] = 'batch_insert_merge'
entity['test'] = True
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
batch = [('upsert', entity, {'mode':UpdateMode.MERGE})]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' in transaction_result[0]
entity = await self.table.get_entity('001', 'batch_insert_merge')
assert entity is not None
assert 'value' == entity['test2']
assert 1234567890 == entity['test4']
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_delete(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = u'001'
entity['RowKey'] = u'batch_delete'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = u'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(partition_key=u'001', row_key=u'batch_delete')
assert 3 == entity['test3']
batch = [('delete', entity)]
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert 'etag' not in transaction_result[0]
with pytest.raises(ResourceNotFoundError):
entity = await self.table.get_entity(partition_key=entity['PartitionKey'], row_key=entity['RowKey'])
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_inserts(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = 'batch_inserts'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
transaction_count = 0
batch = []
for i in range(100):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, transaction_count)
assert 'etag' in transaction_result[0]
entities = self.table.query_entities("PartitionKey eq 'batch_inserts'")
length = 0
async for e in entities:
length += 1
# Assert
assert entities is not None
assert 100 == length
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_all_operations_together(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
# Act
entity = TableEntity()
entity['PartitionKey'] = '003'
entity['RowKey'] = 'batch_all_operations_together-1'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
entity['test5'] = datetime.utcnow()
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-2'
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-3'
await self.table.create_entity(entity)
entity['RowKey'] = 'batch_all_operations_together-4'
await self.table.create_entity(entity)
transaction_count = 0
batch = []
entity['RowKey'] = 'batch_all_operations_together'
batch.append((TransactionOperation.CREATE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-1'
batch.append((TransactionOperation.DELETE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-2'
entity['test3'] = 10
batch.append((TransactionOperation.UPDATE, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-3'
entity['test3'] = 100
batch.append((TransactionOperation.UPDATE, entity.copy(), {'mode': UpdateMode.REPLACE}))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-4'
entity['test3'] = 10
batch.append((TransactionOperation.UPSERT, entity.copy()))
transaction_count += 1
entity['RowKey'] = 'batch_all_operations_together-5'
batch.append((TransactionOperation.UPSERT, entity.copy(), {'mode': UpdateMode.REPLACE}))
transaction_count += 1
transaction_result = await self.table.submit_transaction(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, transaction_count)
assert 'etag' in transaction_result[0]
assert 'etag' not in transaction_result[1]
assert 'etag' in transaction_result[2]
assert 'etag' in transaction_result[3]
assert 'etag' in transaction_result[4]
assert 'etag' in transaction_result[5]
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
assert 5 == length
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_same_row_operations_fail(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = []
entity = self._create_updated_entity_dict(
'001', 'batch_negative_1')
batch.append(('update', entity.copy()))
entity = self._create_random_entity_dict(
'001', 'batch_negative_1')
batch.append(('update', entity.copy(), {'mode': UpdateMode.REPLACE}))
# Assert
with pytest.raises(TableTransactionError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_different_partition_operations_fail(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = []
entity = self._create_updated_entity_dict(
'001', 'batch_negative_1')
batch.append(('update', entity.copy()))
entity = self._create_random_entity_dict(
'002', 'batch_negative_1')
batch.append(('update', entity.copy()))
# Assert
with pytest.raises(ValueError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@tables_decorator_async
async def test_batch_too_many_ops(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
with pytest.raises(TableTransactionError):
batch = []
for i in range(0, 101):
entity = TableEntity()
entity['PartitionKey'] = 'large'
entity['RowKey'] = 'item{0}'.format(i)
batch.append(('create', entity.copy()))
await self.table.submit_transaction(batch)
# Assert
finally:
await self._tear_down()
@tables_decorator_async
async def test_new_non_existent_table(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
tc = self.ts.get_table_client("doesntexist")
batch = [('create', entity)]
with pytest.raises(TableTransactionError):
resp = await tc.submit_transaction(batch)
# Assert
finally:
await self._tear_down()
@tables_decorator_async
async def test_new_invalid_key(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
invalid_key = tables_primary_storage_account_key[0:-6] + "==" # cut off a bit from the end to invalidate
key_list = list(tables_primary_storage_account_key)
key_list[-6:] = list("0000==")
invalid_key = ''.join(key_list)
self.ts = TableServiceClient(self.account_url(tables_storage_account_name, "table"), invalid_key)
self.table_name = self.get_resource_name('uttable')
self.table = self.ts.get_table_client(self.table_name)
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = [('create', entity)]
with pytest.raises(ClientAuthenticationError):
resp = await self.table.submit_transaction(batch)
@tables_decorator_async
async def test_new_delete_nonexistent_entity(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = [('delete', entity)]
with pytest.raises(TableTransactionError):
resp = await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@pytest.mark.live_test_only
@tables_decorator_async
async def test_batch_sas_auth(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
token = generate_table_sas(
tables_storage_account_name,
tables_primary_storage_account_key,
self.table_name,
permission=TableSasPermissions(add=True, read=True, update=True, delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1),
)
token = AzureSasCredential(token)
# Act
service = TableServiceClient(
self.account_url(tables_storage_account_name, "table"),
credential=token,
)
table = service.get_table_client(self.table_name)
entity = TableEntity()
entity['PartitionKey'] = 'batch_inserts'
entity['test'] = EntityProperty(True, EdmType.BOOLEAN)
entity['test2'] = 'value'
entity['test3'] = 3
entity['test4'] = EntityProperty(1234567890, EdmType.INT32)
batch = []
transaction_count = 0
for i in range(10):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
transaction_count += 1
transaction_result = await table.submit_transaction(batch)
assert transaction_result is not None
total_entities = 0
async for e in table.list_entities():
total_entities += 1
assert total_entities == transaction_count
finally:
await self._tear_down()
@pytest.mark.live_test_only # Request bodies are very large
@tables_decorator_async
async def test_batch_request_too_large(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
from azure.data.tables import RequestTooLargeError
try:
batch = []
entity = {
'PartitionKey': 'pk001',
'Foo': os.urandom(1024*64),
'Bar': os.urandom(1024*64),
'Baz': os.urandom(1024*64)
}
for i in range(50):
entity['RowKey'] = str(i)
batch.append(('create', entity.copy()))
with pytest.raises(RequestTooLargeError):
await self.table.submit_transaction(batch)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@tables_decorator_async
async def test_delete_batch_with_bad_kwarg(self, tables_storage_account_name, tables_primary_storage_account_key):
# Arrange
await self._set_up(tables_storage_account_name, tables_primary_storage_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
received = await self.table.get_entity(entity["PartitionKey"], entity["RowKey"])
good_etag = received.metadata["etag"]
received.metadata["etag"] = u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"'
batch = [('delete', received, {"match_condition": MatchConditions.IfNotModified})]
with pytest.raises(TableTransactionError):
await self.table.submit_transaction(batch)
received.metadata["etag"] = good_etag
batch = [('delete', received, {"match_condition": MatchConditions.IfNotModified})]
resp = await self.table.submit_transaction(batch)
assert resp is not None
finally:
await self._tear_down()
class StorageTableBatchTest(AzureTestCase, AsyncTableTestCase):
async def _set_up(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
self.ts = TableServiceClient(
self.account_url(tables_cosmos_account_name, "cosmos"),
tables_primary_cosmos_account_key)
self.table_name = self.get_resource_name('uttable')
self.table = self.ts.get_table_client(self.table_name)
if self.is_live:
try:
await self.ts.create_table(self.table_name)
except ResourceExistsError:
pass
self.test_tables = []
async def _tear_down(self):
if self.is_live:
try:
await self.ts.delete_table(self.table_name)
except:
pass
for table_name in self.test_tables:
try:
await self.ts.delete_table(table_name)
except:
pass
sleep(SLEEP_DELAY)
await self.table.close()
#--Helpers-----------------------------------------------------------------
def _get_table_reference(self, prefix=TEST_TABLE_PREFIX):
table_name = self.get_resource_name(prefix)
self.test_tables.append(table_name)
return self.ts.get_table_client(table_name)
def _create_pk_rk(self, pk, rk):
try:
pk = pk if pk is not None else self.get_resource_name('pk').decode(
'utf-8')
rk = rk if rk is not None else self.get_resource_name('rk').decode(
'utf-8')
except AttributeError:
pk = pk if pk is not None else self.get_resource_name('pk')
rk = rk if rk is not None else self.get_resource_name('rk')
return pk, rk
def _create_random_entity_dict(self, pk=None, rk=None):
"""
Creates a dictionary-based entity with fixed values, using all
of the supported data types.
"""
# partition = pk if pk is not None else self.get_resource_name('pk').decode('utf-8')
# row = rk if rk is not None else self.get_resource_name('rk').decode('utf-8')
partition, row = self._create_pk_rk(pk, rk)
properties = {
'PartitionKey': partition,
'RowKey': row,
'age': 39,
'sex': u'male',
'married': True,
'deceased': False,
'optional': None,
'ratio': 3.1,
'evenratio': 3.0,
'large': 933311100,
'Birthday': datetime(1973, 10, 4, tzinfo=tzutc()),
'birthday': datetime(1970, 10, 4, tzinfo=tzutc()),
'binary': b'binary',
'other': EntityProperty(value=20, type=EdmType.INT32),
'clsid': uuid.UUID('c9da6455-213d-42c9-9a79-3e9149a57833')
}
return TableEntity(**properties)
def _create_updated_entity_dict(self, partition, row):
'''
Creates a dictionary-based entity with fixed values, with a
different set of values than the default entity. It
adds fields, changes field values, changes field types,
and removes fields when compared to the default entity.
'''
return {
'PartitionKey': partition,
'RowKey': row,
'age': u'abc',
'sex': u'female',
'sign': u'aquarius',
'birthday': datetime(1991, 10, 4, tzinfo=tzutc())
}
def _assert_default_entity(self, entity):
'''
Asserts that the entity passed in matches the default entity.
'''
assert entity['age'] == 39
assert entity['sex'] == 'male'
assert entity['married'] == True
assert entity['deceased'] == False
assert not "optional" in entity
assert entity['ratio'] == 3.1
assert entity['evenratio'] == 3.0
assert entity['large'] == 933311100
assert entity['Birthday'] == datetime(1973, 10, 4, tzinfo=tzutc())
assert entity['birthday'] == datetime(1970, 10, 4, tzinfo=tzutc())
assert entity['binary'].value == b'binary'
assert entity['other'] == 20
assert entity['clsid'] == uuid.UUID(
'c9da6455-213d-42c9-9a79-3e9149a57833')
assert '_metadata' in entity
def _assert_updated_entity(self, entity):
'''
Asserts that the entity passed in matches the updated entity.
'''
assert entity.age == 'abc'
assert entity.sex == 'female'
assert not hasattr(entity, "married")
assert not hasattr(entity, "deceased")
assert entity.sign == 'aquarius'
assert not hasattr(entity, "optional")
assert not hasattr(entity, "ratio")
assert not hasattr(entity, "evenratio")
assert not hasattr(entity, "large")
assert not hasattr(entity, "Birthday")
assert entity.birthday, datetime(1991, 10, 4, tzinfo=tzutc())
assert not hasattr(entity, "other")
assert not hasattr(entity, "clsid")
assert entity['_metadata']['etag'] is not None
def _assert_valid_batch_transaction(self, transaction, length):
assert isinstance(transaction, BatchTransactionResult)
assert length == len(transaction.entities)
assert length == len(transaction.results)
assert length == len(transaction.requests)
#--Test cases for batch ---------------------------------------------
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_single_insert(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.create_entity(entity)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
sent_entity = transaction_result.get_entity(entity.RowKey)
assert sent_entity is not None
e = await self.table.get_entity(row_key=entity.RowKey,
partition_key=entity.PartitionKey)
assert e.test == entity.test.value
assert e.test2 == entity.test2
assert e.test3 == entity.test3
assert e.test4 == entity.test4.value
assert sent_entity['test'] == entity.test.value
assert sent_entity['test2'] == entity.test2
assert sent_entity['test3'] == entity.test3
assert sent_entity['test4'] == entity.test4.value
finally:
await self._tear_down()
@pytest.mark.skip(
"merge operations fail in cosmos: https://github.com/Azure/azure-sdk-for-python/issues/13844"
)
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_single_update(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
resp = await self.table.create_entity(entity)
assert resp is not None
entity.test3 = 5
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.update_entity(entity, mode=UpdateMode.MERGE)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert transaction_result.get_entity(entity.RowKey) is not None
result = await self.table.get_entity(
row_key=entity.RowKey, partition_key=entity.PartitionKey)
assert result.PartitionKey == u'001'
assert result.RowKey == u'batch_insert'
assert result.test3.value == 5
finally:
await self._tear_down()
@pytest.mark.skip(
"merge operations fail in cosmos: https://github.com/Azure/azure-sdk-for-python/issues/13844"
)
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_update(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_update'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(u'001', u'batch_update')
assert 3 == entity.test3.value
entity.test2 = u'value1'
batch = self.table.create_batch()
batch.update_entity(entity)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert transaction_result.get_entity(entity.RowKey) is not None
result = await self.table.get_entity('001', 'batch_update')
assert 'value1' == result.test2.value
assert entity.PartitionKey == u'001'
assert entity.RowKey == u'batch_update'
finally:
await self._tear_down()
@pytest.mark.skip(
"merge operations fail in cosmos: https://github.com/Azure/azure-sdk-for-python/issues/13844"
)
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_merge(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_merge'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
assert 3 == entity.test3
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_merge'
entity.test2 = u'value1'
batch = self.table.create_batch()
batch.update_entity(entity, mode=UpdateMode.MERGE)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert transaction_result.get_entity(entity.RowKey) is not None
resp_entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_merge')
assert entity.test2 == resp_entity.test2.value
assert 1234567890 == resp_entity.test4.value
assert entity.PartitionKey == resp_entity.PartitionKey
assert entity.RowKey == resp_entity.RowKey
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_update_if_match(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity=entity)
etag = resp['etag']
# Act
sent_entity = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = self.table.create_batch()
batch.update_entity(sent_entity,
etag=etag,
match_condition=MatchConditions.IfNotModified,
mode=UpdateMode.REPLACE)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert transaction_result.get_entity(
sent_entity['RowKey']) is not None
entity = await self.table.get_entity(
partition_key=entity['PartitionKey'], row_key=entity['RowKey'])
self._assert_updated_entity(entity)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_update_if_doesnt_match(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict()
resp = await self.table.create_entity(entity)
assert resp is not None
# Act
sent_entity1 = self._create_updated_entity_dict(
entity['PartitionKey'], entity['RowKey'])
batch = self.table.create_batch()
batch.update_entity(
sent_entity1,
etag=u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"',
match_condition=MatchConditions.IfNotModified)
with pytest.raises(HttpResponseError):
await self.table.send_batch(batch)
# Assert
received_entity = await self.table.get_entity(
entity['PartitionKey'], entity['RowKey'])
self._assert_default_entity(received_entity)
finally:
await self._tear_down()
@pytest.mark.skip(
"merge operations fail in cosmos: https://github.com/Azure/azure-sdk-for-python/issues/13844"
)
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_insert_replace(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert_replace'
entity.test = True
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.upsert_entity(entity)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert transaction_result.get_entity(entity.RowKey) is not None
entity = await self.table.get_entity('001', 'batch_insert_replace')
assert entity is not None
assert 'value' == entity.test2.value
assert 1234567890 == entity.test4.value
finally:
await self._tear_down()
@pytest.mark.skip(
"merge operations fail in cosmos: https://github.com/Azure/azure-sdk-for-python/issues/13844"
)
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_insert_merge(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert_merge'
entity.test = True
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
batch = self.table.create_batch()
batch.upsert_entity(entity, mode=UpdateMode.MERGE)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert transaction_result.get_entity(entity.RowKey) is not None
entity = await self.table.get_entity('001', 'batch_insert_merge')
assert entity is not None
assert 'value' == entity.test2.value
assert 1234567890 == entity.test4.value
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_delete(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = u'001'
entity.RowKey = u'batch_delete'
entity.test = EntityProperty(True)
entity.test2 = u'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity = await self.table.get_entity(partition_key=u'001',
row_key=u'batch_delete')
assert 3 == entity.test3
batch = self.table.create_batch()
batch.delete_entity(partition_key=entity.PartitionKey,
row_key=entity.RowKey)
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result, 1)
assert transaction_result.get_entity(entity.RowKey) is not None
with pytest.raises(ResourceNotFoundError):
entity = await self.table.get_entity(
partition_key=entity.PartitionKey, row_key=entity.RowKey)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_inserts(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = 'batch_inserts'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
transaction_count = 0
batch = self.table.create_batch()
for i in range(20):
entity.RowKey = str(i)
batch.create_entity(entity)
transaction_count += 1
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert transaction_result.get_entity(entity.RowKey) is not None
entities = self.table.query_entities(
"PartitionKey eq 'batch_inserts'")
length = 0
async for e in entities:
length += 1
# Assert
assert entities is not None
assert transaction_count == length
finally:
await self._tear_down()
@pytest.mark.skip(
"merge operations fail in cosmos: https://github.com/Azure/azure-sdk-for-python/issues/13844"
)
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_all_operations_together(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '003'
entity.RowKey = 'batch_all_operations_together-1'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-2'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
await self.table.create_entity(entity)
transaction_count = 0
batch = self.table.create_batch()
entity.RowKey = 'batch_all_operations_together'
batch.create_entity(entity)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-1'
batch.delete_entity(entity.PartitionKey, entity.RowKey)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-2'
entity.test3 = 10
batch.update_entity(entity)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-3'
entity.test3 = 100
batch.update_entity(entity, mode=UpdateMode.MERGE)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-4'
entity.test3 = 10
batch.upsert_entity(entity)
transaction_count += 1
entity.RowKey = 'batch_all_operations_together-5'
batch.upsert_entity(entity, mode=UpdateMode.MERGE)
transaction_count += 1
transaction_result = await self.table.send_batch(batch)
# Assert
self._assert_valid_batch_transaction(transaction_result,
transaction_count)
assert transaction_result.get_entity(
'batch_all_operations_together') is not None
assert transaction_result.get_entity(
'batch_all_operations_together-1') is not None
assert transaction_result.get_entity(
'batch_all_operations_together-2') is not None
assert transaction_result.get_entity(
'batch_all_operations_together-3') is not None
assert transaction_result.get_entity(
'batch_all_operations_together-4') is not None
assert transaction_result.get_entity(
'batch_all_operations_together-5') is not None
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
assert 5 == length
finally:
await self._tear_down()
@pytest.mark.skip(
"merge operations fail in cosmos: https://github.com/Azure/azure-sdk-for-python/issues/13844"
)
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_all_operations_together_context_manager(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
# Act
entity = TableEntity()
entity.PartitionKey = '003'
entity.RowKey = 'batch_all_operations_together-1'
entity.test = EntityProperty(True)
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(1234567890)
entity.test5 = datetime.utcnow()
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-2'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
await self.table.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
await self.table.create_entity(entity)
async with self.table.create_batch() as batch:
entity.RowKey = 'batch_all_operations_together'
batch.create_entity(entity)
entity.RowKey = 'batch_all_operations_together-1'
batch.delete_entity(entity.PartitionKey, entity.RowKey)
entity.RowKey = 'batch_all_operations_together-2'
entity.test3 = 10
batch.update_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
entity.test3 = 100
batch.update_entity(entity, mode=UpdateMode.MERGE)
entity.RowKey = 'batch_all_operations_together-4'
entity.test3 = 10
batch.upsert_entity(entity)
entity.RowKey = 'batch_all_operations_together-5'
batch.upsert_entity(entity, mode=UpdateMode.MERGE)
# Assert
entities = self.table.query_entities("PartitionKey eq '003'")
length = 0
async for e in entities:
length += 1
assert 4 == length
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_same_row_operations_fail(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = self.table.create_batch()
entity = self._create_updated_entity_dict('001',
'batch_negative_1')
batch.update_entity(entity)
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch.update_entity(entity)
# Assert
with pytest.raises(BatchErrorException):
await self.table.send_batch(batch)
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_batch_different_partition_operations_fail(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
batch = self.table.create_batch()
entity = self._create_updated_entity_dict('001',
'batch_negative_1')
batch.update_entity(entity)
entity = self._create_random_entity_dict('002', 'batch_negative_1')
# Assert
with pytest.raises(ValueError):
batch.create_entity(entity)
finally:
await self._tear_down()
@pytest.mark.skip("On Cosmos, the limit is not specified.")
@CosmosPreparer()
async def test_batch_too_many_ops(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
await self.table.create_entity(entity)
# Act
with pytest.raises(BatchErrorException):
batch = self.table.create_batch()
for i in range(0, 101):
entity = TableEntity()
entity.PartitionKey = 'large'
entity.RowKey = 'item{0}'.format(i)
batch.create_entity(entity)
# Assert
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_new_non_existent_table(self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
tc = self.ts.get_table_client("doesntexist")
batch = tc.create_batch()
batch.create_entity(entity)
with pytest.raises(ResourceNotFoundError):
resp = await tc.send_batch(batch)
# Assert
finally:
await self._tear_down()
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
@CosmosPreparer()
async def test_new_delete_nonexistent_entity(
self, tables_cosmos_account_name,
tables_primary_cosmos_account_key):
# Arrange
await self._set_up(tables_cosmos_account_name,
tables_primary_cosmos_account_key)
try:
entity = self._create_random_entity_dict('001', 'batch_negative_1')
batch = self.table.create_batch()
batch.delete_entity(entity['PartitionKey'], entity['RowKey'])
with pytest.raises(ResourceNotFoundError):
resp = await self.table.send_batch(batch)
finally:
await self._tear_down()
class StorageRetryTest(AzureTestCase, AsyncTableTestCase):
async def _set_up(self,
tables_storage_account_name,
tables_primary_storage_account_key,
url='table',
default_table=True,
**kwargs):
self.table_name = self.get_resource_name('uttable')
self.ts = TableServiceClient(
self.account_url(tables_storage_account_name, url),
credential=tables_primary_storage_account_key,
**kwargs)
self.table = self.ts.get_table_client(self.table_name)
if self.is_live and default_table:
try:
await self.ts.create_table(self.table_name)
except ResourceExistsError:
pass
self.query_tables = []
async def _tear_down(self, **kwargs):
if self.is_live:
try:
await self.ts.delete_table(self.table_name, **kwargs)
except:
pass
try:
for table_name in self.query_tables:
try:
await self.ts.delete_table(table_name, **kwargs)
except:
pass
except AttributeError:
pass
# --Test Cases --------------------------------------------
@TablesPreparer()
async def test_retry_on_server_error_async(
self, tables_storage_account_name,
tables_primary_storage_account_key):
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
default_table=False)
try:
callback = ResponseCallback(status=201,
new_status=500).override_status
new_table_name = self.get_resource_name('uttable')
# The initial create will return 201, but we overwrite it with 500 and retry.
# The retry will then get a 409 conflict.
with pytest.raises(ResourceExistsError):
await self.ts.create_table(new_table_name,
raw_response_hook=callback)
finally:
await self.ts.delete_table(new_table_name)
await self._tear_down()
@TablesPreparer()
async def test_retry_on_timeout_async(self, tables_storage_account_name,
tables_primary_storage_account_key):
retry = ExponentialRetry(initial_backoff=1, increment_base=2)
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
retry_policy=retry,
default_table=False)
new_table_name = self.get_resource_name('uttable')
callback = ResponseCallback(status=201, new_status=408).override_status
try:
# The initial create will return 201, but we overwrite it with 408 and retry.
# The retry will then get a 409 conflict.
with pytest.raises(ResourceExistsError):
await self.ts.create_table(new_table_name,
raw_response_hook=callback)
finally:
await self.ts.delete_table(new_table_name)
await self._tear_down()
@TablesPreparer()
async def test_retry_callback_and_retry_context_async(
self, tables_storage_account_name,
tables_primary_storage_account_key):
retry = LinearRetry(backoff=1)
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
retry_policy=retry,
default_table=False)
new_table_name = self.get_resource_name('uttable')
callback = ResponseCallback(status=201, new_status=408).override_status
def assert_exception_is_present_on_retry_context(**kwargs):
self.assertIsNotNone(kwargs.get('response'))
self.assertEqual(kwargs['response'].status_code, 408)
try:
# The initial create will return 201, but we overwrite it with 408 and retry.
# The retry will then get a 409 conflict.
with pytest.raises(ResourceExistsError):
await self.ts.create_table(
new_table_name,
raw_response_hook=callback,
retry_hook=assert_exception_is_present_on_retry_context)
finally:
await self.ts.delete_table(new_table_name)
await self._tear_down()
@pytest.mark.live_test_only
@TablesPreparer()
async def test_retry_on_socket_timeout_async(
self, tables_storage_account_name,
tables_primary_storage_account_key):
retry = LinearRetry(backoff=1)
retry_transport = RetryAioHttpTransport(connection_timeout=11,
read_timeout=0.000000000001)
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
retry_policy=retry,
transport=retry_transport,
default_table=False)
new_table_name = self.get_resource_name('uttable')
try:
with pytest.raises(AzureError) as error:
await self.ts.create_table(new_table_name)
# 3 retries + 1 original == 4
assert retry_transport.count == 4
# This call should succeed on the server side, but fail on the client side due to socket timeout
self.assertTrue(
'Timeout on reading' in str(error.value),
'Expected socket timeout but got different exception.')
finally:
# TODO: Why can I not just reset the connection timeout???
await self._set_up(tables_storage_account_name,
tables_primary_storage_account_key,
default_table=False)
# we must make the timeout normal again to let the delete operation succeed
await self.ts.delete_table(new_table_name)
await self._tear_down()
