def dbAzureApp(figurename, isinserted):
account_name = 'shanistorage'
account_key = 'j1COI4eq+p/Yl/e8dVCAiaHX/ly1StLuFAlgalNhVI+rjU8YL6wkWlulld4XIZ/5kjnrqkFyGhQsVo68y9NWpg=='
account = CloudStorageAccount(account_name, account_key)
table_service = None
the_figures=[]
try:
table_service = account.create_table_service()
table_name = 'azureFirstStor'
#insret to a list by order
by_ord=0
for entity in table_service.query_entities(table_name):
the_figures.insert(by_ord,entity['NameFigure'])
by_ord +=1
#insert into the DB
if isinserted is True:
str_coun= str(by_ord)
part_key= 'N' + str_coun
figure_new = {'PartitionKey': part_key, 'RowKey': str_coun, 'NameFigure' : figurename}
# Insert the entity into the table
table_service.insert_entity(table_name, figure_new)
the_figures.insert(by_ord,figurename)
# delete an entity if want
# table_service.delete_entity(table_name, part_key, str_coun)
except Exception as e:
print('Error occurred in the sample. Please make sure the account name and key are correct.', e)
return the_figures
def retrieve_symbols():
# retrieves the list of option symbols from the MS SQL Server database. This has been changed
# to use Azure Table Storage, but is kept here for reference in case we need to later
# make a call to the older MSSQL database
#
# modified to retrieve symbols from the Azure table storage 'Symbols' table
# Old code to insert / update the table in the MSSQL database. This code has been replaced
###with pypyodbc.connect('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') as connection:
### # retrieve the cursor
### cursor = connection.cursor()
### cursor.execute("Select Symbol from Symbols")
### rows = cursor.fetchall()
### #connection.close() # connection is automatically closed when using the
### #"with" block
### return rows
account_name = config.STORAGE_ACCOUNT_NAME
account_key = config.STORAGE_ACCOUNT_KEY
account = CloudStorageAccount(account_name, account_key)
table_service = None
try:
table_service = account.create_table_service()
symbols_list = table_service.query_entities(
ut.TABLE_NAME_SYMBOLS, filter="PartitionKey eq 'Symbol'")
return symbols_list
except Exception as e:
print(
'Error occurred in the sample. If you are using the emulator, please make sure the emulator is running.',
e)
def get_storage_account_details(subscription_id, creds, resource_group_name,
account_name):
storage_client = StorageManagementClient(creds, subscription_id)
account_result = storage_client.storage_accounts.get_properties(
resource_group_name,
account_name,
)
storage_account_keys = storage_client.storage_accounts.list_keys(
resource_group_name,
account_name,
)
account_key = storage_account_keys.key1
account = CloudStorageAccount(account_name, account_key)
blob_service = account.create_blob_service()
file_service = account.create_file_service()
queue_service = account.create_queue_service()
table_service = account.create_table_service()
model = StorageAccountDetails()
model.account_props = account_result
model.account_keys = storage_account_keys
model.blob_containers = blob_service.iterate_containers()
model.queues = queue_service.iterate_queues()
#TODO: find out why listing shares doesn't work
#model.shares = file_service.iterate_shares()
model.shares = []
model.tables = table_service.iterate_tables()
model.blob_service_properties = blob_service.get_blob_service_properties()
model.queue_service_properties = queue_service.get_queue_service_properties(
)
model.table_service_properties = table_service.get_table_service_properties(
)
return model
def get_storage_account_details(creds, resource_group_name, account_name):
storage_client = StorageManagementClient(creds)
account_result = storage_client.storage_accounts.get_properties(resource_group_name, account_name)
keys_result = storage_client.storage_accounts.list_keys(resource_group_name, account_name)
account_key = keys_result.storage_account_keys.key1
account = CloudStorageAccount(account_name, account_key)
blob_service = account.create_blob_service()
file_service = account.create_file_service()
queue_service = account.create_queue_service()
table_service = account.create_table_service()
model = StorageAccountDetails()
model.account_props = account_result.storage_account
model.account_keys = keys_result.storage_account_keys
model.blob_containers = blob_service.iterate_containers()
model.queues = queue_service.iterate_queues()
# TODO: find out why listing shares doesn't work
# model.shares = file_service.iterate_shares()
model.shares = []
model.tables = table_service.iterate_tables()
model.blob_service_properties = blob_service.get_blob_service_properties()
model.queue_service_properties = queue_service.get_queue_service_properties()
model.table_service_properties = table_service.get_table_service_properties()
return model
class CloudStorageAccountTest(AzureTestCase):
def setUp(self):
self.account = CloudStorageAccount(
account_name=credentials.getStorageServicesName(),
account_key=credentials.getStorageServicesKey())
#--Test cases --------------------------------------------------------
def test_create_blob_service(self):
# Arrange
# Act
service = self.account.create_blob_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, BlobService)
self.assertEqual(service.account_name,
credentials.getStorageServicesName())
self.assertEqual(service.account_key,
credentials.getStorageServicesKey())
def test_create_blob_service_empty_credentials(self):
# Arrange
# Act
bad_account = CloudStorageAccount('', '')
with self.assertRaises(WindowsAzureError):
service = bad_account.create_blob_service()
# Assert
def test_create_table_service(self):
# Arrange
# Act
service = self.account.create_table_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, TableService)
self.assertEqual(service.account_name,
credentials.getStorageServicesName())
self.assertEqual(service.account_key,
credentials.getStorageServicesKey())
def test_create_queue_service(self):
# Arrange
# Act
service = self.account.create_queue_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, QueueService)
self.assertEqual(service.account_name,
credentials.getStorageServicesName())
self.assertEqual(service.account_key,
credentials.getStorageServicesKey())
class CloudStorageAccountTest(AzureTestCase):
def setUp(self):
self.account = CloudStorageAccount(
account_name=credentials.getStorageServicesName(),
account_key=credentials.getStorageServicesKey())
#--Test cases --------------------------------------------------------
def test_create_blob_service(self):
# Arrange
# Act
service = self.account.create_blob_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, BlobService)
self.assertEqual(service.account_name,
credentials.getStorageServicesName())
self.assertEqual(service.account_key,
credentials.getStorageServicesKey())
def test_create_blob_service_empty_credentials(self):
# Arrange
# Act
bad_account = CloudStorageAccount('', '')
with self.assertRaises(WindowsAzureError):
service = bad_account.create_blob_service()
# Assert
def test_create_table_service(self):
# Arrange
# Act
service = self.account.create_table_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, TableService)
self.assertEqual(service.account_name,
credentials.getStorageServicesName())
self.assertEqual(service.account_key,
credentials.getStorageServicesKey())
def test_create_queue_service(self):
# Arrange
# Act
service = self.account.create_queue_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, QueueService)
self.assertEqual(service.account_name,
credentials.getStorageServicesName())
self.assertEqual(service.account_key,
credentials.getStorageServicesKey())
class StorageAccountTest(ExtendedTestCase):
def setUp(self):
self.account_name = 'storagename'
self.account_key = 'NzhL3hKZbJBuJ2484dPTR+xF30kYaWSSCbs2BzLgVVI1woqeST/1IgqaLm6QAOTxtGvxctSNbIR/1hW8yH+bJg=='
self.account = CloudStorageAccount(self.account_name, self.account_key)
#--Test cases --------------------------------------------------------
def test_create_blob_service(self):
# Arrange
# Act
service = self.account.create_blob_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, BlobService)
self.assertEqual(service.account_name, self.account_name)
self.assertEqual(service.account_key, self.account_key)
def test_create_blob_service_empty_credentials(self):
# Arrange
# Act
bad_account = CloudStorageAccount('', '')
with self.assertRaises(ValueError):
service = bad_account.create_blob_service()
# Assert
def test_create_table_service(self):
# Arrange
# Act
service = self.account.create_table_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, TableService)
self.assertEqual(service.account_name, self.account_name)
self.assertEqual(service.account_key, self.account_key)
def test_create_queue_service(self):
# Arrange
# Act
service = self.account.create_queue_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, QueueService)
self.assertEqual(service.account_name, self.account_name)
self.assertEqual(service.account_key, self.account_key)
class StorageAccountTest(StorageTestCase):
def setUp(self):
self.account_name = 'storagename'
self.account_key = 'NzhL3hKZbJBuJ2484dPTR+xF30kYaWSSCbs2BzLgVVI1woqeST/1IgqaLm6QAOTxtGvxctSNbIR/1hW8yH+bJg=='
self.account = CloudStorageAccount(self.account_name, self.account_key)
#--Test cases --------------------------------------------------------
def test_create_blob_service(self):
# Arrange
# Act
service = self.account.create_blob_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, BlobService)
self.assertEqual(service.account_name, self.account_name)
self.assertEqual(service.account_key, self.account_key)
def test_create_blob_service_empty_credentials(self):
# Arrange
# Act
bad_account = CloudStorageAccount('', '')
with self.assertRaises(ValueError):
service = bad_account.create_blob_service()
# Assert
def test_create_table_service(self):
# Arrange
# Act
service = self.account.create_table_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, TableService)
self.assertEqual(service.account_name, self.account_name)
self.assertEqual(service.account_key, self.account_key)
def test_create_queue_service(self):
# Arrange
# Act
service = self.account.create_queue_service()
# Assert
self.assertIsNotNone(service)
self.assertIsInstance(service, QueueService)
self.assertEqual(service.account_name, self.account_name)
self.assertEqual(service.account_key, self.account_key)
def _turn_on_minute_metrics(self, name, key,
metric_table_retention_period):
account = CloudStorageAccount(account_name=name,
account_key=key,
sas_token=None)
retention_policy = RetentionPolicy(enabled=True,
days=metric_table_retention_period)
metrics = Metrics(enabled=True,
include_apis=True,
retention_policy=retention_policy)
table_service = account.create_table_service()
table_service.set_table_service_properties(minute_metrics=metrics)
blob_service = account.create_page_blob_service()
blob_service.set_blob_service_properties(minute_metrics=metrics)
file_service = account.create_file_service()
file_service.set_file_service_properties(minute_metrics=metrics)
queue_service = account.create_queue_service()
queue_service.set_queue_service_properties(minute_metrics=metrics)
def append_css(r, subreddit, css, position, height):
new_css = '.flair-' + str(position) + '{background-position: 0 -' + str(
height * position) + 'px}'
r.set_stylesheet(subreddit, css + new_css)
def log(message):
table_service.insert_entity('logs',
{'PartitionKey': 'flair', 'RowKey': str(datetime.datetime.now()),
'text': message})
print('[*] ' + message)
storage_account = CloudStorageAccount(storage_account_name, storage_account_key)
table_service = storage_account.create_table_service()
blob_service = storage_account.create_block_blob_service()
blob_service.create_container('images', public_access='container')
table_service.create_table('flair')
table_service.create_table('logs')
r = praw.Reddit(user_agent)
r.login(username, password)
r.config.decode_html_entities = True
while True:
for message in (m for m in r.get_unread(limit=None)):
log('received mesage from ' + message.author.name)
try:
file, text = get_flair_info(message)
class TableStorageConnector:
def __init__(self, config):
self.config = config
self.account = CloudStorageAccount(
account_name=config.storage_account_name,
account_key=config.storage_account_key)
self.service = self.account.create_table_service()
#only successful table creations (name is unique, not None, and does not contain illegal characters) returns the original table_name
def create_table(self, table_name):
if (table_name == None):
return None
success = self.service.create_table(table_name)
if (success == False):
return None
return table_name
#only successful table deletions (table exists) returns the original table_name
def delete_table(self, table_name):
if (table_name == None):
return None
success = self.service.delete_table(table_name)
if (success == False):
return None
return table_name
#only returns true if name is not None and the table exists
def exists(self, table_name):
if (table_name == None):
return False
return self.service.exists(table_name)
# All operations in the same batch must have the same partition key but different row keys
# Batches can hold from 1 to 100 entities
# Batches are atomic. All operations completed simulatenously. If one operation fails, they all fail.
# Insert, update, merge, insert or merge, insert or replace, and delete entity operations are supported
def _batch_upload(self, table_name, post_entities):
# Context manager style
if (len(post_entities) == 0):
return
if (len(post_entities) > 100):
raise ValueError("batch cannot be over 100 entries")
# Context manager style
# with self.service.batch(table_name) as batch:
# for entity in post_entities:
# batch.insert_entity(entity)
# Commit style
batch = TableBatch()
for entity in post_entities:
if (entity == None):
print "entity none"
break
batch.insert_entity(entity)
self.service.commit_batch(table_name, batch)
def upload_instagram_post_entities(self, brand_name, IPE):
print "Batch upload called for brand: " + brand_name
table_name = brand_name
if not (self.exists(table_name)):
table_name = self.create_table(table_name)
print(table_name, " table created...")
current_index = 0
while (True):
pk = self.get_pk()
indices = [(current_index + i) for i in range(100)]
instagram_post_entities = [
IPE.posts[i] for i in indices if (i < len(IPE.posts))
]
if (len(instagram_post_entities) == 0):
break
current_index += len(instagram_post_entities)
batch = [
self.create_entity(post, pk=pk, rk=self.get_rk())
for post in instagram_post_entities
]
# print batch
self._batch_upload(table_name, batch)
print "uploading batch with pk " + pk
print "batch upload completed"
def get_pk(self):
return 'pk{}'.format(str(uuid.uuid4()).replace('-', ''))
def get_rk(self):
return 'rk{}'.format(str(uuid.uuid4()).replace('-', ''))
def create_entity(self, instagram_post_entity, pk=None, rk=None):
entity = {}
if (pk != None):
entity['PartitionKey'] = pk
else:
entity['PartitionKey'] = self.get_pk()
if (rk != None):
entity['RowKey'] = rk
else:
entity['RowKey'] = self.get_pk()
if (PICTURE_ID in instagram_post_entity
and instagram_post_entity[PICTURE_ID] != None):
entity[PICTURE_ID] = EntityProperty(
EdmType.INT64, instagram_post_entity[PICTURE_ID])
if (OWNER_ID in instagram_post_entity
and instagram_post_entity[OWNER_ID] != None):
entity[OWNER_ID] = EntityProperty(EdmType.INT64,
instagram_post_entity[OWNER_ID])
if (LOGO_NAME in instagram_post_entity
and instagram_post_entity[LOGO_NAME] != None):
entity[LOGO_NAME] = EntityProperty(
EdmType.STRING, instagram_post_entity[LOGO_NAME])
#This is a unix epoch timestamp (we will do conversion from utc to epoch for search)
if (TIME in instagram_post_entity
and instagram_post_entity[TIME] != None):
entity[TIME] = EntityProperty(EdmType.INT64,
instagram_post_entity[TIME])
if (CAPTION in instagram_post_entity
and instagram_post_entity[CAPTION] != None):
entity[CAPTION] = EntityProperty(EdmType.STRING,
instagram_post_entity[CAPTION])
if (TAGS in instagram_post_entity
and instagram_post_entity[TAGS] != None):
entity[TAGS] = EntityProperty(
EdmType.STRING,
self.serialize_entity_attribute_value(
instagram_post_entity[TAGS]))
if (HAS_LOGO in instagram_post_entity
and instagram_post_entity[HAS_LOGO] != None):
entity[HAS_LOGO] = EntityProperty(EdmType.BOOLEAN,
instagram_post_entity[HAS_LOGO])
if (ACCURACY in instagram_post_entity
and instagram_post_entity[ACCURACY] != None):
entity[ACCURACY] = EntityProperty(EdmType.DOUBLE,
instagram_post_entity[ACCURACY])
if (IMAGE_CONTEXT in instagram_post_entity
and instagram_post_entity[IMAGE_CONTEXT] != None):
entity[IMAGE_CONTEXT] = EntityProperty(
EdmType.STRING,
self.serialize_entity_attribute_value(
instagram_post_entity[IMAGE_CONTEXT]))
if (IMAGE_PATH in instagram_post_entity
and instagram_post_entity[IMAGE_PATH] != None):
entity[IMAGE_PATH] = EntityProperty(
EdmType.STRING, instagram_post_entity[IMAGE_PATH])
if (DIMENSIONS in instagram_post_entity
and instagram_post_entity[DIMENSIONS] != None):
entity[DIMENSIONS] = EntityProperty(
EdmType.STRING,
self.serialize_entity_attribute_value(
instagram_post_entity[DIMENSIONS]))
return entity
def serialize_entity_attribute_value(self, attribute_value):
if (attribute_value == None):
return ""
return json.dumps(attribute_value, indent=4, ensure_ascii=False)
def get_all_entries(self, table_name):
return list(self.service.query_entities(table_name))
# blob services.
#
# See http://go.microsoft.com/fwlink/?linkid=246933 for Storage documentation.
#
STORAGE_ACCOUNT_NAME = '__paste_your_storage_account_name_here__'
STORAGE_ACCOUNT_KEY = '__paste_your_storage_key_here__'
if os.environ.get('EMULATED', '').lower() == 'true':
# Running in the emulator, so use the development storage account
storage_account = CloudStorageAccount(None, None)
else:
storage_account = CloudStorageAccount(STORAGE_ACCOUNT_NAME,
STORAGE_ACCOUNT_KEY)
blob_service = storage_account.create_blob_service()
table_service = storage_account.create_table_service()
queue_service = storage_account.create_queue_service()
#
# Service Bus is a messaging solution for applications. It sits between
# components of your applications and enables them to exchange messages in a
# loosely coupled way for improved scale and resiliency.
#
# See http://go.microsoft.com/fwlink/?linkid=246934 for Service Bus documentation.
#
SERVICE_BUS_NAMESPACE = '__paste_your_service_bus_namespace_here__'
SERVICE_BUS_KEY = '__paste_your_service_bus_key_here__'
bus_service = ServiceBusService(SERVICE_BUS_NAMESPACE,
SERVICE_BUS_KEY,
issuer='owner')
class StorageAccountTest(StorageTestCase):
def setUp(self):
super(StorageAccountTest, self).setUp()
self.account_name = self.settings.STORAGE_ACCOUNT_NAME
self.account_key = self.settings.STORAGE_ACCOUNT_KEY
self.sas_token = '?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D'
self.account = CloudStorageAccount(self.account_name, self.account_key)
#--Helpers-----------------------------------------------------------------
def validate_service(self, service, type):
self.assertIsNotNone(service)
self.assertIsInstance(service, type)
self.assertEqual(service.account_name, self.account_name)
self.assertEqual(service.account_key, self.account_key)
#--Test cases --------------------------------------------------------
def test_create_block_blob_service(self):
# Arrange
# Act
service = self.account.create_block_blob_service()
# Assert
self.validate_service(service, BlockBlobService)
def test_create_page_blob_service(self):
# Arrange
# Act
service = self.account.create_page_blob_service()
# Assert
self.validate_service(service, PageBlobService)
def test_create_append_blob_service(self):
# Arrange
# Act
service = self.account.create_append_blob_service()
# Assert
self.validate_service(service, AppendBlobService)
def test_create_table_service(self):
# Arrange
# Act
service = self.account.create_table_service()
# Assert
self.validate_service(service, TableService)
def test_create_queue_service(self):
# Arrange
# Act
service = self.account.create_queue_service()
# Assert
self.validate_service(service, QueueService)
def test_create_file_service(self):
# Arrange
# Act
service = self.account.create_file_service()
# Assert
self.validate_service(service, FileService)
def test_create_service_no_key(self):
# Arrange
# Act
bad_account = CloudStorageAccount('', '')
with self.assertRaises(ValueError):
service = bad_account.create_block_blob_service()
# Assert
def test_create_account_sas(self):
# Arrange
# Act
sas_account = CloudStorageAccount(self.account_name, sas_token=self.sas_token)
service = sas_account.create_block_blob_service()
# Assert
self.assertIsNotNone(service)
self.assertEqual(service.account_name, self.account_name)
self.assertIsNone(service.account_key)
self.assertEqual(service.sas_token, self.sas_token)
def test_create_account_sas_and_key(self):
# Arrange
# Act
account = CloudStorageAccount(self.account_name, self.account_key, self.sas_token)
service = account.create_block_blob_service()
# Assert
self.validate_service(service, BlockBlobService)
def test_create_account_emulated(self):
# Arrange
# Act
account = CloudStorageAccount(is_emulated=True)
service = account.create_block_blob_service()
# Assert
self.assertIsNotNone(service)
self.assertEqual(service.account_name, 'devstoreaccount1')
self.assertIsNotNone(service.account_key)
@record
def test_generate_account_sas(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recordingfile(self.test_mode):
return
# Arrange
token = self.account.generate_shared_access_signature(
Services.BLOB,
ResourceTypes.OBJECT,
AccountPermissions.READ,
datetime.utcnow() + timedelta(hours=1),
)
service = self.account.create_block_blob_service()
data = b'shared access signature with read permission on blob'
container_name='container1'
blob_name = 'blob1.txt'
try:
service.create_container(container_name)
service.create_blob_from_bytes(container_name, blob_name, data)
# Act
url = service.make_blob_url(
container_name,
blob_name,
sas_token=token,
)
response = requests.get(url)
# Assert
self.assertTrue(response.ok)
self.assertEqual(data, response.content)
finally:
service.delete_container(container_name)
class StorageTableContext():
"""Initializes the repository with the specified settings dict.
Required settings in config dict are:
- AZURE_STORAGE_NAME
- AZURE_STORAGE_KEY
- AZURE_KEY_IDENTIFIER
- AZURE_SECRET_KEY
- AZURE_STORAGE_IS_EMULATED
"""
_account = None
_account_name = ''
_account_key = ''
_is_emulated = False
_kek = None
_key_resolver = None
_modeldefinitions = []
REQUIRED = True
# decorators
def getmodeldefinition(self, storageobject, required=False):
""" find modeldefinition for StorageTableModel or StorageTableQuery """
if isinstance(storageobject, StorageTableModel):
definitionlist = [
definition for definition in self._modeldefinitions
if definition['modelname'] == storageobject.__class__.__name__
]
elif isinstance(storageobject, StorageTableQuery):
""" StorageTableQuery """
storagemodel = storageobject._storagemodel
definitionlist = [
definition for definition in self._modeldefinitions
if definition['modelname'] == storagemodel.__class__.__name__
]
else:
raise Exception(
"Argument is not an StorageTableModel nor an StorageTableQuery"
)
# is there only one modeldefinition ?
# hopefully!
modeldefinition = None
if len(definitionlist) == 1:
modeldefinition = definitionlist[0]
elif len(definitionlist) > 1:
raise ModelRegisteredMoreThanOnceError(storageobject)
# is there a modeldefinition if required ?
if required and modeldefinition is None:
raise ModelNotRegisteredError(storageobject)
return modeldefinition
# constructor
def __init__(self, **kwargs):
""" parse kwargs """
self._account_name = kwargs.get('AZURE_STORAGE_NAME', '')
self._account_key = kwargs.get('AZURE_STORAGE_KEY', '')
self._is_emulated = kwargs.get('AZURE_STORAGE_IS_EMULATED', False)
self._key_identifier = kwargs.get('AZURE_KEY_IDENTIFIER', '')
self._secret_key = kwargs.get('AZURE_SECRET_KEY', '')
""" account init """
if self._is_emulated:
self._account = CloudStorageAccount(is_emulated=True)
elif self._account_name != '' and self._account_key != '':
self._account = CloudStorageAccount(self._account_name,
self._account_key)
else:
raise AzureException
""" init table model list """
self._modeldefinitions = []
def __createtable__(self, modeldefinition: dict) -> bool:
if (not modeldefinition['tableservice'] is None):
try:
modeldefinition['tableservice'].create_table(
modeldefinition['tablename'])
return True
except Exception as e:
log.error('failed to create {} with error {}'.format(
tablename, e))
return False
else:
return False
pass
def __deletetable__(self, modeldefinition: dict) -> bool:
if (not modeldefinition['tableservice'] is None):
try:
modeldefinition['tableservice'].delete_table(
modeldefinition['tablename'])
return True
except Exception as e:
msg = 'failed to create {} with error {}'.format(tablename, e)
raise AzureStorageWrapException(msg=msg)
else:
return False
pass
def register_model(self, storagemodel: object):
""" set up an Tableservice for an StorageTableModel in your Azure Storage Account
Will create the Table if not exist!
required Parameter is:
- storagemodel: StorageTableModel(Object)
"""
modeldefinition = self.getmodeldefinition(storagemodel, False)
if modeldefinition is None:
""" test if queuename already exists """
if [
model for model in self._modeldefinitions
if model['tablename'] == storagemodel._tablename
]:
raise NameConventionError(storagemodel._tablename)
""" test if queuename fits to azure naming rules """
if not test_azurestorage_nameconventions(storagemodel._tablename,
'StorageTableModel'):
raise NameConventionError(storagemodel._tablename)
""" now register model """
modeldefinition = {
'modelname': storagemodel.__class__.__name__,
'tablename': storagemodel._tablename,
'encrypt': storagemodel._encrypt,
'tableservice': self._account.create_table_service()
}
if modeldefinition['encrypt']:
""" encrypt init """
# Create the KEK used for encryption.
# KeyWrapper is the provided sample implementation, but the user may use their own object as long as it implements the interface above.
kek = KeyWrapper(self._key_identifier,
self._secret_key) # Key identifier
# Create the key resolver used for decryption.
# KeyResolver is the provided sample implementation, but the user may use whatever implementation they choose so long as the function set on the service object behaves appropriately.
key_resolver = KeyResolver()
key_resolver.put_key(kek)
# Create the EncryptionResolver Function to determine Properties to en/decrypt
encryptionresolver = self.__encryptionresolver__(
modeldefinition['encrypt'])
# Set the require Encryption, KEK and key resolver on the service object.
modeldefinition['tableservice'].key_encryption_key = kek
modeldefinition[
'tableservice'].key_resolver_funcion = key_resolver.resolve_key
modeldefinition[
'tableservice'].encryption_resolver_function = encryptionresolver
pass
self.__createtable__(modeldefinition)
self._modeldefinitions.append(modeldefinition)
log.info(
'model {} registered successfully. Models are {!s}.'.format(
modeldefinition['modelname'],
[model['modelname'] for model in self._modeldefinitions]))
else:
log.info('model {} already registered. Models are {!s}.'.format(
modeldefinition['modelname'],
[model['modelname'] for model in self._modeldefinitions]))
def unregister_model(self, storagemodel: object, delete_table=False):
""" clear up an Tableservice for an StorageTableModel in your Azure Storage Account
Will delete the Table if delete_table Flag is True!
required Parameter is:
- storagemodel: StorageTableModel(Object)
Optional Parameter is:
- delete_table: bool
"""
# get modeldefinition
modeldefinition = self.getmodeldefinition(storagemodel, True)
# remove from modeldefinitions
for i in range(len(self._modeldefinitions)):
if self._modeldefinitions[i]['modelname'] == modeldefinition[
'modelname']:
del self._modeldefinitions[i]
break
# delete table from storage if delete_table == True
if delete_table:
self.__deletetable__(modeldefinition)
pass
# methods
def exists(self, storagemodel) -> bool:
modeldefinition = self.getmodeldefinition(storagemodel, True)
exists = False
if storagemodel._exists is None:
try:
pk = storagemodel.getPartitionKey()
rk = storagemodel.getRowKey()
entity = modeldefinition['tableservice'].get_entity(
modeldefinition['tablename'], pk, rk)
storagemodel._exists = True
exists = True
except AzureMissingResourceHttpError:
storagemodel._exists = False
except Exception as e:
msg = 'failed to test {} with error {}'.format(
modeldefinition['tablename'], e)
raise AzureStorageWrapException(msg=msg)
else:
exists = storagemodel._exists
return exists
def get(self, storagemodel) -> StorageTableModel:
""" load entity data from storage to vars in self """
modeldefinition = self.getmodeldefinition(storagemodel, True)
try:
pk = storagemodel.getPartitionKey()
rk = storagemodel.getRowKey()
entity = modeldefinition['tableservice'].get_entity(
modeldefinition['tablename'], pk, rk)
storagemodel._exists = True
""" sync with entity values """
for key, default in vars(storagemodel).items():
if not key.startswith('_') and key not in [
'', 'PartitionKey', 'RowKey'
]:
value = getattr(entity, key, None)
if not value is None:
setattr(storagemodel, key, value)
except AzureMissingResourceHttpError as e:
log.debug(
'can not get table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'
.format(modeldefinition['tablename'], pk, rk, e))
storagemodel._exists = False
except Exception as e:
msg = 'can not get table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format(
modeldefinition['tablename'], pk, rk, e)
raise AzureStorageWrapException(msg=msg)
finally:
return storagemodel
def insert(self, storagemodel) -> StorageTableModel:
""" insert model into storage """
modeldefinition = self.getmodeldefinition(storagemodel, True)
try:
modeldefinition['tableservice'].insert_or_replace_entity(
modeldefinition['tablename'], storagemodel.entity())
storagemodel._exists = True
except AzureMissingResourceHttpError as e:
storagemodel._exists = False
log.debug(
'can not insert or replace table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'
.format(modeldefinition['tablename'],
storagemodel.getPartitionKey(),
storagemodel.getRowKey(), e))
except Exception as e:
storagemodel._exists = False
msg = 'can not insert or replace table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format(
modeldefinition['tablename'], storagemodel.PartitionKey,
storagemodel.RowKey, e)
raise AzureStorageWrapException(msg=msg)
finally:
return storagemodel
def merge(self, storagemodel) -> StorageTableModel:
""" try to merge entry """
modeldefinition = self.getmodeldefinition(storagemodel, True)
try:
pk = storagemodel.getPartitionKey()
rk = storagemodel.getRowKey()
entity = modeldefinition['tableservice'].get_entity(
modeldefinition['tablename'], pk, rk)
""" merge with entity values """
for key, default in vars(storagemodel.__class__).items():
if not key.startswith('_') and key not in ['']:
if isinstance(default, PartitionKey) or isinstance(
default, RowKey) or isinstance(
default, EncryptKey):
default = default._default
newvalue = getattr(storagemodel, key, None)
if (newvalue is None) or (newvalue == default):
oldvalue = getattr(entity, key, default)
setattr(storagemodel, key, oldvalue)
modeldefinition['tableservice'].insert_or_replace_entity(
modeldefinition['tablename'], storagemodel.entity())
storagemodel._exists = True
except AzureMissingResourceHttpError as e:
log.debug(
'can not merge table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'
.format(modeldefinition['tablename'], pk, rk, e))
except Exception as e:
log.debug(
'can not merge table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'
.format(modeldefinition['tablename'], pk, rk, e))
finally:
return storagemodel
def delete(self, storagemodel):
""" delete existing Entity """
modeldefinition = self.getmodeldefinition(storagemodel, True)
pk = storagemodel.getPartitionKey()
rk = storagemodel.getRowKey()
try:
modeldefinition['tableservice'].delete_entity(
modeldefinition['tablename'], pk, rk)
storagemodel._exists = False
except AzureMissingResourceHttpError as e:
log.debug(
'can not delete table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'
.format(modeldefinition['tablename'], pk, rk, e))
finally:
return storagemodel
def query(self, storagequery) -> StorageTableQuery:
modeldefinition = self.getmodeldefinition(storagequery, True)
try:
if (not storagequery._select is None) and (storagequery._select !=
''):
storagequery.extend(
modeldefinition['tableservice'].query_entities(
modeldefinition['tablename'],
filter=storagequery._queryfilter,
select=storagequery._select))
else:
storagequery.extend(
modeldefinition['tableservice'].query_entities(
modeldefinition['tablename'],
filter=storagequery._queryfilter))
except AzureMissingResourceHttpError as e:
storagequery = []
log.debug(
'can not query table {} with filters {} because {!s}'.format(
modeldefinition['tablename'], storagequery._queryfilter,
e))
except Exception as e:
msg = 'can not query table {} with filters {} because {!s}'.format(
modeldefinition['tablename'], storagequery._queryfilter, e)
raise AzureStorageWrapException(msg=msg)
return storagequery
def table_isempty(self, tablename, PartitionKey='', RowKey='') -> bool:
if (not self._tableservice is None):
filter = "PartitionKey eq '{}'".format(
PartitionKey) if PartitionKey != '' else ''
if filter == '':
filter = "RowKey eq '{}'".format(
RowKey) if RowKey != '' else ''
else:
filter = filter + ("and RowKey eq '{}'".format(RowKey)
if RowKey != '' else '')
try:
entities = list(modeldefinition['tableservice'].query_entities(
tablename,
filter=filter,
select='PartitionKey',
num_results=1))
if len(entities) == 1:
return False
else:
return True
except AzureMissingResourceHttpError as e:
log.debug('failed to query {} with error {}'.format(
tablename, e))
return True
except Exception as e:
msg = '{!s}'.format(e)
raise AzureStorageWrapException(msg=msg)
else:
return True
pass
def __encryptionresolver__(self, encryptproperties):
def encryptionresolver(pk, rk, property_name):
if property_name in encryptproperties:
return True
else:
return False
return encryptionresolver
def Init(name,key):
global account, table_service
account = CloudStorageAccount(name,key)
print("storage account created")
table_service = account.create_table_service()
print("table service created")
self.muscleTargeted = muscleTargeted
self.url = url
baseUrl = "https://www.bodybuilding.com"
url = baseUrl + "/exercises/finder"
page = 1
workToDo = True
exercises = []
batch = TableBatch()
account = CloudStorageAccount(is_emulated=True)
table_service = account.create_table_service()
table_service.create_table('exercises')
counter = 0
while (workToDo):
headers = {"x-bb-ex-skip-finder-form": "YES"}
urlToGet = url + "/" + str(page)
print(urlToGet)
r = requests.get(urlToGet, headers=headers)
soup = BeautifulSoup(r.text, "html.parser")
exerciseDivs = soup.findAll("div", {"class": "ExResult-row"})