def _create_random_entity_class(self, pk=None, rk=None):
'''
Creates a class-based entity with fixed values, using all
of the supported data types.
'''
partition = pk if pk is not None else self.get_resource_name('pk')
row = rk if rk is not None else self.get_resource_name('rk')
entity = Entity()
entity.PartitionKey = partition
entity.RowKey = row
entity.age = 39
entity.sex = 'male'
entity.name = 'John Doe'
entity.married = True
entity.deceased = False
entity.optional = None
entity.evenratio = 3.0
entity.ratio = 3.1
entity.large = 933311100
entity.Birthday = datetime(1973, 10, 4)
entity.birthday = datetime(1970, 10, 4)
entity.binary = EntityProperty(EdmType.BINARY, b'binary')
entity.other = EntityProperty(EdmType.INT32, 20)
entity.clsid = EntityProperty(
EdmType.GUID, 'c9da6455-213d-42c9-9a79-3e9149a57833')
return entity
def test_query_entities_large(self):
# Arrange
table_name = self._create_query_table(0)
total_entities_count = 1000
entities_per_batch = 50
for j in range(total_entities_count // entities_per_batch):
batch = TableBatch()
for i in range(entities_per_batch):
entity = Entity()
entity.PartitionKey = 'large'
entity.RowKey = 'batch{0}-item{1}'.format(j, i)
entity.test = EntityProperty(EdmType.BOOLEAN, 'true')
entity.test2 = 'hello world;' * 100
entity.test3 = 3
entity.test4 = EntityProperty(EdmType.INT64, '1234567890')
entity.test5 = datetime(2016, 12, 31, 11, 59, 59, 0)
batch.insert_entity(entity)
self.ts.commit_batch(table_name, batch)
# Act
start_time = datetime.now()
entities = list(self.ts.query_entities(table_name))
elapsed_time = datetime.now() - start_time
# Assert
print('query_entities took {0} secs.'.format(
elapsed_time.total_seconds()))
# azure allocates 5 seconds to execute a query
# if it runs slowly, it will return fewer results and make the test fail
self.assertEqual(len(entities), total_entities_count)
def create_entity_dict(self):
'''
Creates a dict-based entity with fixed values, using all of the supported data types.
'''
entity = {}
# Partition key and row key must be strings and are required
entity['PartitionKey'] = 'pk{}'.format(
str(uuid.uuid4()).replace('-', ''))
entity['RowKey'] = 'rk{}'.format(str(uuid.uuid4()).replace('-', ''))
# Some basic types are inferred
entity['age'] = 39 # EdmType.INT64
entity['large'] = 933311100 # EdmType.INT64
entity['sex'] = 'male' # EdmType.STRING
entity['married'] = True # EdmType.BOOLEAN
entity['ratio'] = 3.1 # EdmType.DOUBLE
entity['birthday'] = datetime(1970, 10, 4) # EdmType.DATETIME
# Binary, Int32 and GUID must be explicitly typed
entity['binary'] = EntityProperty(EdmType.BINARY, b'xyz')
entity['other'] = EntityProperty(EdmType.INT32, 20)
entity['clsid'] = EntityProperty(
EdmType.GUID, 'c9da6455-213d-42c9-9a79-3e9149a57833')
return entity
def test_batch_insert_merge(self):
# Arrange
# Act
entity = Entity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_insert_merge'
entity.test = EntityProperty(EdmType.BOOLEAN, 'true')
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(EdmType.INT64, '1234567890')
entity.test5 = datetime.utcnow()
batch = TableBatch()
batch.insert_or_merge_entity(entity)
resp = self.ts.commit_batch(self.table_name, batch)
# Assert
self.assertIsNotNone(resp)
entity = self.ts.get_entity(self.table_name, '001',
'batch_insert_merge')
self.assertIsNotNone(entity)
self.assertEqual('value', entity.test2)
self.assertEqual(1234567890, entity.test4)
self.assertEqual(resp[0], entity.etag)
def _associateFileWithJob(self, jobStoreFileID, jobStoreID=None):
if jobStoreID is not None:
self.jobFileIDs.insert_entity(
entity={
'PartitionKey': EntityProperty('Edm.String', jobStoreID),
'RowKey': EntityProperty('Edm.String', jobStoreFileID)
})
def put_encrypted_entity_properties(self):
table_name = self._create_table()
# Can use a dict or the Entity class to encrypt entities.
# The EntityProperty object takes an optional parameteter, 'encrypt'
# that marks the property for encryption when set to true.
entity1 = self._create_base_entity_dict()
entity1['foo'] = EntityProperty(EdmType.STRING, 'bar', True)
entity2 = self._create_base_entity_class()
entity2.foo = EntityProperty(EdmType.STRING, 'bar', True)
entity3 = self._create_base_entity_class()
entity3['badValue'] = EntityProperty(EdmType.INT64, 12, True)
entity4 = self._create_base_entity_class()
# KeyWrapper implements the key encryption key interface outlined
# in the insert/get entity documentation.
# Setting this property will tell these APIs to encrypt the entity.
self.service.key_encryption_key = KeyWrapper('key1')
self.service.insert_entity(table_name, entity1)
self.service.insert_entity(table_name, entity2)
# Note: The internal encryption process requires two properties, so there
# are only 250 custom properties available when encrypting.
# Note: str is the only type valid for encryption. Trying to encrypt other
# properties will throw.
self.service.delete_table(table_name)
def _create_default_entity_dict(self, pk=None, rk=None):
'''
Creates a dictionary-based entity with fixed values, using all
of the supported data types.
'''
partition = pk if pk is not None else self.get_resource_name('pk')
row = rk if rk is not None else self.get_resource_name('rk')
return {'PartitionKey': partition,
'RowKey': row,
'age': 39,
'sex': 'male',
'name': 'John Doe',
'married': True,
'deceased': False,
'optional': None,
'ratio': 3.1,
'evenratio': 3.0,
'large': 933311100,
'Birthday': datetime(1973, 10, 4),
'birthday': datetime(1970, 10, 4),
'binary': EntityProperty(EdmType.BINARY, b'binary'),
'other': EntityProperty(EdmType.INT32, 20),
'clsid': EntityProperty(
EdmType.GUID,
'c9da6455-213d-42c9-9a79-3e9149a57833')}
def test_batch_reuse(self):
# Arrange
table2 = self._get_table_reference('table2')
self.ts.create_table(table2)
# Act
entity = Entity()
entity.PartitionKey = '003'
entity.RowKey = 'batch_all_operations_together-1'
entity.test = EntityProperty(EdmType.BOOLEAN, 'true')
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(EdmType.INT64, '1234567890')
entity.test5 = datetime.utcnow()
batch = TableBatch()
batch.insert_entity(entity)
entity.RowKey = 'batch_all_operations_together-2'
batch.insert_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
batch.insert_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
batch.insert_entity(entity)
self.ts.commit_batch(self.table_name, batch)
self.ts.commit_batch(table2, batch)
batch = TableBatch()
entity.RowKey = 'batch_all_operations_together'
batch.insert_entity(entity)
entity.RowKey = 'batch_all_operations_together-1'
batch.delete_entity(entity.PartitionKey, entity.RowKey)
entity.RowKey = 'batch_all_operations_together-2'
entity.test3 = 10
batch.update_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
entity.test3 = 100
batch.merge_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
entity.test3 = 10
batch.insert_or_replace_entity(entity)
entity.RowKey = 'batch_all_operations_together-5'
batch.insert_or_merge_entity(entity)
self.ts.commit_batch(self.table_name, batch)
resp = self.ts.commit_batch(table2, batch)
# Assert
self.assertEqual(6, len(resp))
entities = list(
self.ts.query_entities(self.table_name, "PartitionKey eq '003'",
''))
self.assertEqual(5, len(entities))
def test_insert_entity_with_large_int64_value_throws(self):
# Arrange
# Act
dict64 = self._create_random_base_entity_dict()
dict64['large'] = EntityProperty(EdmType.INT64, 2 ** 63)
# Assert
with self.assertRaisesRegexp(TypeError,
'{0} is too large to be cast to type Edm.Int64.'.format(2 ** 63)):
self.ts.insert_entity(self.table_name, dict64)
dict64['large'] = EntityProperty(EdmType.INT64, -(2 ** 63 + 1))
with self.assertRaisesRegexp(TypeError,
'{0} is too large to be cast to type Edm.Int64.'.format(-(2 ** 63 + 1))):
self.ts.insert_entity(self.table_name, dict64)
def test_batch_all_operations_together_context_manager(self):
# Arrange
# Act
entity = Entity()
entity.PartitionKey = '003'
entity.RowKey = 'batch_all_operations_together-1'
entity.test = EntityProperty(EdmType.BOOLEAN, 'true')
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(EdmType.INT64, '1234567890')
entity.test5 = datetime.utcnow()
self.ts.insert_entity(self.table_name, entity)
entity.RowKey = 'batch_all_operations_together-2'
self.ts.insert_entity(self.table_name, entity)
entity.RowKey = 'batch_all_operations_together-3'
self.ts.insert_entity(self.table_name, entity)
entity.RowKey = 'batch_all_operations_together-4'
self.ts.insert_entity(self.table_name, entity)
with self.ts.batch(self.table_name) as batch:
entity.RowKey = 'batch_all_operations_together'
batch.insert_entity(entity)
entity.RowKey = 'batch_all_operations_together-1'
batch.delete_entity(entity.PartitionKey, entity.RowKey)
entity.RowKey = 'batch_all_operations_together-2'
entity.test3 = 10
batch.update_entity(entity)
entity.RowKey = 'batch_all_operations_together-3'
entity.test3 = 100
batch.merge_entity(entity)
entity.RowKey = 'batch_all_operations_together-4'
entity.test3 = 10
batch.insert_or_replace_entity(entity)
entity.RowKey = 'batch_all_operations_together-5'
batch.insert_or_merge_entity(entity)
# Assert
entities = list(
self.ts.query_entities(self.table_name, "PartitionKey eq '003'",
''))
self.assertEqual(5, len(entities))
def test_get_encrypted_dict(self):
# Arrange
self.ts.require_encryption = True
entity = self._create_default_entity_dict()
entity['sex'] = EntityProperty(EdmType.STRING, entity['sex'], True)
self.ts.key_encryption_key = KeyWrapper('key1')
self.ts.insert_entity(self.table_name, entity)
# Act
new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey'])
# Assert
self._assert_default_entity(new_entity)
def test_batch_inserts(self):
# Arrange
# Act
entity = Entity()
entity.PartitionKey = 'batch_inserts'
entity.test = EntityProperty(EdmType.BOOLEAN, 'true')
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(EdmType.INT64, '1234567890')
batch = TableBatch()
for i in range(100):
entity.RowKey = str(i)
batch.insert_entity(entity)
self.ts.commit_batch(self.table_name, batch)
entities = list(self.ts.query_entities(self.table_name, "PartitionKey eq 'batch_inserts'", ''))
# Assert
self.assertIsNotNone(entities)
self.assertEqual(100, len(entities))
def test_replace_entity(self):
# Arrange
entity = self._create_random_entity_class()
self.ts.insert_entity(self.table_name, entity)
entity['sex'] = EntityProperty(EdmType.STRING, 'female', True)
self.ts.key_encryption_key = KeyWrapper('key1')
# Act
self.ts.require_encryption = True
self.ts.update_entity(self.table_name, entity)
new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey'])
# Assert
self.assertEqual(new_entity['sex'], entity['sex'].value)
def test_binary_property_value(self):
# Arrange
binary_data = b'\x01\x02\x03\x04\x05\x06\x07\x08\t\n'
entity = self._create_random_base_entity_dict()
entity.update({'binary': EntityProperty(EdmType.BINARY, binary_data)})
# Act
self.ts.insert_entity(self.table_name, entity)
resp = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey'])
# Assert
self.assertIsNotNone(resp)
self.assertEqual(resp.binary.type, EdmType.BINARY)
self.assertEqual(resp.binary.value, binary_data)
def test_batch_delete(self):
# Arrange
# Act
entity = Entity()
entity.PartitionKey = '001'
entity.RowKey = 'batch_delete'
entity.test = EntityProperty(EdmType.BOOLEAN, 'true')
entity.test2 = 'value'
entity.test3 = 3
entity.test4 = EntityProperty(EdmType.INT64, '1234567890')
entity.test5 = datetime.utcnow()
self.ts.insert_entity(self.table_name, entity)
entity = self.ts.get_entity(self.table_name, '001', 'batch_delete')
self.assertEqual(3, entity.test3)
batch = TableBatch()
batch.delete_entity('001', 'batch_delete')
resp = self.ts.commit_batch(self.table_name, batch)
# Assert
self.assertIsNotNone(resp)
self.assertIsNone(resp[0])
def toEntity(self, chunkSize=maxAzureTablePropertySize):
"""
:param chunkSize: the size of a chunk for splitting up the serialized job into chunks
that each fit into a property value of the an Azure table entity
:rtype: dict
"""
assert chunkSize <= maxAzureTablePropertySize
item = {}
serializedAndEncodedJob = bz2.compress(pickle.dumps(self, protocol=pickle.HIGHEST_PROTOCOL))
jobChunks = [serializedAndEncodedJob[i:i + chunkSize]
for i in range(0, len(serializedAndEncodedJob), chunkSize)]
for attributeOrder, chunk in enumerate(jobChunks):
item['_' + str(attributeOrder).zfill(3)] = EntityProperty('Edm.Binary', chunk)
item['RowKey'] = str(self.jobStoreID)
item['PartitionKey'] = str(AzureTable.defaultPartition)
return Entity(item)
def test_insert_encrypt_invalid_types(self):
# Arrange
self.ts.require_encryption = True
entity_binary = self._create_random_entity_class()
entity_binary['bytes'] = EntityProperty(EdmType.BINARY, urandom(10),
True)
entity_boolean = self._create_random_entity_class()
entity_boolean['married'] = EntityProperty(EdmType.BOOLEAN, True, True)
entity_date_time = self._create_random_entity_class()
entity_date_time['birthday'] = EntityProperty(
EdmType.DATETIME, entity_date_time['birthday'], True)
entity_double = self._create_random_entity_class()
entity_double['ratio'] = EntityProperty(EdmType.DATETIME,
entity_double['ratio'], True)
entity_guid = self._create_random_entity_class()
entity_guid['clsid'].encrypt = True
entity_int32 = self._create_random_entity_class()
entity_int32['other'].encrypt = True
entity_int64 = self._create_random_entity_class()
entity_int64['large'] = EntityProperty(EdmType.INT64,
entity_int64['large'], True)
self.ts.key_encryption_key = KeyWrapper('key1')
entity_none_str = self._create_random_entity_class()
entity_none_str['none_str'] = EntityProperty(EdmType.STRING, None,
True)
# Act
# Assert
try:
self.ts.insert_entity(self.table_name, entity_binary)
self.fail()
except ValueError as e:
self.assertEqual(str(e), _ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION)
with self.assertRaises(ValueError):
self.ts.insert_entity(self.table_name, entity_boolean)
with self.assertRaises(ValueError):
self.ts.insert_entity(self.table_name, entity_date_time)
with self.assertRaises(ValueError):
self.ts.insert_entity(self.table_name, entity_double)
with self.assertRaises(ValueError):
self.ts.insert_entity(self.table_name, entity_guid)
with self.assertRaises(ValueError):
self.ts.insert_entity(self.table_name, entity_int32)
with self.assertRaises(ValueError):
self.ts.insert_entity(self.table_name, entity_int64)
with self.assertRaises(ValueError):
self.ts.insert_entity(self.table_name, entity_none_str)
def _create_entity_for_encryption(self):
entity = self._create_base_entity_class()
entity['foo'] = EntityProperty(EdmType.STRING, 'bar', True)
return entity
def _create_default_entity_for_encryption(self):
entity = self._create_random_entity_class()
entity['sex'] = EntityProperty(EdmType.STRING, entity['sex'], True)
entity['name'] = EntityProperty(EdmType.STRING, entity['name'], True)
return entity
