def dbAzureApp(figurename, isinserted): account_name = 'shanistorage' account_key = 'j1COI4eq+p/Yl/e8dVCAiaHX/ly1StLuFAlgalNhVI+rjU8YL6wkWlulld4XIZ/5kjnrqkFyGhQsVo68y9NWpg==' account = CloudStorageAccount(account_name, account_key) table_service = None the_figures=[] try: table_service = account.create_table_service() table_name = 'azureFirstStor' #insret to a list by order by_ord=0 for entity in table_service.query_entities(table_name): the_figures.insert(by_ord,entity['NameFigure']) by_ord +=1 #insert into the DB if isinserted is True: str_coun= str(by_ord) part_key= 'N' + str_coun figure_new = {'PartitionKey': part_key, 'RowKey': str_coun, 'NameFigure' : figurename} # Insert the entity into the table table_service.insert_entity(table_name, figure_new) the_figures.insert(by_ord,figurename) # delete an entity if want # table_service.delete_entity(table_name, part_key, str_coun) except Exception as e: print('Error occurred in the sample. Please make sure the account name and key are correct.', e) return the_figures
def retrieve_symbols(): # retrieves the list of option symbols from the MS SQL Server database. This has been changed # to use Azure Table Storage, but is kept here for reference in case we need to later # make a call to the older MSSQL database # # modified to retrieve symbols from the Azure table storage 'Symbols' table # Old code to insert / update the table in the MSSQL database. This code has been replaced ###with pypyodbc.connect('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') as connection: ### # retrieve the cursor ### cursor = connection.cursor() ### cursor.execute("Select Symbol from Symbols") ### rows = cursor.fetchall() ### #connection.close() # connection is automatically closed when using the ### #"with" block ### return rows account_name = config.STORAGE_ACCOUNT_NAME account_key = config.STORAGE_ACCOUNT_KEY account = CloudStorageAccount(account_name, account_key) table_service = None try: table_service = account.create_table_service() symbols_list = table_service.query_entities( ut.TABLE_NAME_SYMBOLS, filter="PartitionKey eq 'Symbol'") return symbols_list except Exception as e: print( 'Error occurred in the sample. If you are using the emulator, please make sure the emulator is running.', e)
def get_storage_account_details(subscription_id, creds, resource_group_name, account_name): storage_client = StorageManagementClient(creds, subscription_id) account_result = storage_client.storage_accounts.get_properties( resource_group_name, account_name, ) storage_account_keys = storage_client.storage_accounts.list_keys( resource_group_name, account_name, ) account_key = storage_account_keys.key1 account = CloudStorageAccount(account_name, account_key) blob_service = account.create_blob_service() file_service = account.create_file_service() queue_service = account.create_queue_service() table_service = account.create_table_service() model = StorageAccountDetails() model.account_props = account_result model.account_keys = storage_account_keys model.blob_containers = blob_service.iterate_containers() model.queues = queue_service.iterate_queues() #TODO: find out why listing shares doesn't work #model.shares = file_service.iterate_shares() model.shares = [] model.tables = table_service.iterate_tables() model.blob_service_properties = blob_service.get_blob_service_properties() model.queue_service_properties = queue_service.get_queue_service_properties( ) model.table_service_properties = table_service.get_table_service_properties( ) return model
def get_storage_account_details(creds, resource_group_name, account_name): storage_client = StorageManagementClient(creds) account_result = storage_client.storage_accounts.get_properties(resource_group_name, account_name) keys_result = storage_client.storage_accounts.list_keys(resource_group_name, account_name) account_key = keys_result.storage_account_keys.key1 account = CloudStorageAccount(account_name, account_key) blob_service = account.create_blob_service() file_service = account.create_file_service() queue_service = account.create_queue_service() table_service = account.create_table_service() model = StorageAccountDetails() model.account_props = account_result.storage_account model.account_keys = keys_result.storage_account_keys model.blob_containers = blob_service.iterate_containers() model.queues = queue_service.iterate_queues() # TODO: find out why listing shares doesn't work # model.shares = file_service.iterate_shares() model.shares = [] model.tables = table_service.iterate_tables() model.blob_service_properties = blob_service.get_blob_service_properties() model.queue_service_properties = queue_service.get_queue_service_properties() model.table_service_properties = table_service.get_table_service_properties() return model
class CloudStorageAccountTest(AzureTestCase): def setUp(self): self.account = CloudStorageAccount( account_name=credentials.getStorageServicesName(), account_key=credentials.getStorageServicesKey()) #--Test cases -------------------------------------------------------- def test_create_blob_service(self): # Arrange # Act service = self.account.create_blob_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, BlobService) self.assertEqual(service.account_name, credentials.getStorageServicesName()) self.assertEqual(service.account_key, credentials.getStorageServicesKey()) def test_create_blob_service_empty_credentials(self): # Arrange # Act bad_account = CloudStorageAccount('', '') with self.assertRaises(WindowsAzureError): service = bad_account.create_blob_service() # Assert def test_create_table_service(self): # Arrange # Act service = self.account.create_table_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, TableService) self.assertEqual(service.account_name, credentials.getStorageServicesName()) self.assertEqual(service.account_key, credentials.getStorageServicesKey()) def test_create_queue_service(self): # Arrange # Act service = self.account.create_queue_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, QueueService) self.assertEqual(service.account_name, credentials.getStorageServicesName()) self.assertEqual(service.account_key, credentials.getStorageServicesKey())
class StorageAccountTest(ExtendedTestCase): def setUp(self): self.account_name = 'storagename' self.account_key = 'NzhL3hKZbJBuJ2484dPTR+xF30kYaWSSCbs2BzLgVVI1woqeST/1IgqaLm6QAOTxtGvxctSNbIR/1hW8yH+bJg==' self.account = CloudStorageAccount(self.account_name, self.account_key) #--Test cases -------------------------------------------------------- def test_create_blob_service(self): # Arrange # Act service = self.account.create_blob_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, BlobService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) def test_create_blob_service_empty_credentials(self): # Arrange # Act bad_account = CloudStorageAccount('', '') with self.assertRaises(ValueError): service = bad_account.create_blob_service() # Assert def test_create_table_service(self): # Arrange # Act service = self.account.create_table_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, TableService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) def test_create_queue_service(self): # Arrange # Act service = self.account.create_queue_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, QueueService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key)
class StorageAccountTest(StorageTestCase): def setUp(self): self.account_name = 'storagename' self.account_key = 'NzhL3hKZbJBuJ2484dPTR+xF30kYaWSSCbs2BzLgVVI1woqeST/1IgqaLm6QAOTxtGvxctSNbIR/1hW8yH+bJg==' self.account = CloudStorageAccount(self.account_name, self.account_key) #--Test cases -------------------------------------------------------- def test_create_blob_service(self): # Arrange # Act service = self.account.create_blob_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, BlobService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) def test_create_blob_service_empty_credentials(self): # Arrange # Act bad_account = CloudStorageAccount('', '') with self.assertRaises(ValueError): service = bad_account.create_blob_service() # Assert def test_create_table_service(self): # Arrange # Act service = self.account.create_table_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, TableService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) def test_create_queue_service(self): # Arrange # Act service = self.account.create_queue_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, QueueService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key)
def _turn_on_minute_metrics(self, name, key, metric_table_retention_period): account = CloudStorageAccount(account_name=name, account_key=key, sas_token=None) retention_policy = RetentionPolicy(enabled=True, days=metric_table_retention_period) metrics = Metrics(enabled=True, include_apis=True, retention_policy=retention_policy) table_service = account.create_table_service() table_service.set_table_service_properties(minute_metrics=metrics) blob_service = account.create_page_blob_service() blob_service.set_blob_service_properties(minute_metrics=metrics) file_service = account.create_file_service() file_service.set_file_service_properties(minute_metrics=metrics) queue_service = account.create_queue_service() queue_service.set_queue_service_properties(minute_metrics=metrics)
def append_css(r, subreddit, css, position, height): new_css = '.flair-' + str(position) + '{background-position: 0 -' + str( height * position) + 'px}' r.set_stylesheet(subreddit, css + new_css) def log(message): table_service.insert_entity('logs', {'PartitionKey': 'flair', 'RowKey': str(datetime.datetime.now()), 'text': message}) print('[*] ' + message) storage_account = CloudStorageAccount(storage_account_name, storage_account_key) table_service = storage_account.create_table_service() blob_service = storage_account.create_block_blob_service() blob_service.create_container('images', public_access='container') table_service.create_table('flair') table_service.create_table('logs') r = praw.Reddit(user_agent) r.login(username, password) r.config.decode_html_entities = True while True: for message in (m for m in r.get_unread(limit=None)): log('received mesage from ' + message.author.name) try: file, text = get_flair_info(message)
class TableStorageConnector: def __init__(self, config): self.config = config self.account = CloudStorageAccount( account_name=config.storage_account_name, account_key=config.storage_account_key) self.service = self.account.create_table_service() #only successful table creations (name is unique, not None, and does not contain illegal characters) returns the original table_name def create_table(self, table_name): if (table_name == None): return None success = self.service.create_table(table_name) if (success == False): return None return table_name #only successful table deletions (table exists) returns the original table_name def delete_table(self, table_name): if (table_name == None): return None success = self.service.delete_table(table_name) if (success == False): return None return table_name #only returns true if name is not None and the table exists def exists(self, table_name): if (table_name == None): return False return self.service.exists(table_name) # All operations in the same batch must have the same partition key but different row keys # Batches can hold from 1 to 100 entities # Batches are atomic. All operations completed simulatenously. If one operation fails, they all fail. # Insert, update, merge, insert or merge, insert or replace, and delete entity operations are supported def _batch_upload(self, table_name, post_entities): # Context manager style if (len(post_entities) == 0): return if (len(post_entities) > 100): raise ValueError("batch cannot be over 100 entries") # Context manager style # with self.service.batch(table_name) as batch: # for entity in post_entities: # batch.insert_entity(entity) # Commit style batch = TableBatch() for entity in post_entities: if (entity == None): print "entity none" break batch.insert_entity(entity) self.service.commit_batch(table_name, batch) def upload_instagram_post_entities(self, brand_name, IPE): print "Batch upload called for brand: " + brand_name table_name = brand_name if not (self.exists(table_name)): table_name = self.create_table(table_name) print(table_name, " table created...") current_index = 0 while (True): pk = self.get_pk() indices = [(current_index + i) for i in range(100)] instagram_post_entities = [ IPE.posts[i] for i in indices if (i < len(IPE.posts)) ] if (len(instagram_post_entities) == 0): break current_index += len(instagram_post_entities) batch = [ self.create_entity(post, pk=pk, rk=self.get_rk()) for post in instagram_post_entities ] # print batch self._batch_upload(table_name, batch) print "uploading batch with pk " + pk print "batch upload completed" def get_pk(self): return 'pk{}'.format(str(uuid.uuid4()).replace('-', '')) def get_rk(self): return 'rk{}'.format(str(uuid.uuid4()).replace('-', '')) def create_entity(self, instagram_post_entity, pk=None, rk=None): entity = {} if (pk != None): entity['PartitionKey'] = pk else: entity['PartitionKey'] = self.get_pk() if (rk != None): entity['RowKey'] = rk else: entity['RowKey'] = self.get_pk() if (PICTURE_ID in instagram_post_entity and instagram_post_entity[PICTURE_ID] != None): entity[PICTURE_ID] = EntityProperty( EdmType.INT64, instagram_post_entity[PICTURE_ID]) if (OWNER_ID in instagram_post_entity and instagram_post_entity[OWNER_ID] != None): entity[OWNER_ID] = EntityProperty(EdmType.INT64, instagram_post_entity[OWNER_ID]) if (LOGO_NAME in instagram_post_entity and instagram_post_entity[LOGO_NAME] != None): entity[LOGO_NAME] = EntityProperty( EdmType.STRING, instagram_post_entity[LOGO_NAME]) #This is a unix epoch timestamp (we will do conversion from utc to epoch for search) if (TIME in instagram_post_entity and instagram_post_entity[TIME] != None): entity[TIME] = EntityProperty(EdmType.INT64, instagram_post_entity[TIME]) if (CAPTION in instagram_post_entity and instagram_post_entity[CAPTION] != None): entity[CAPTION] = EntityProperty(EdmType.STRING, instagram_post_entity[CAPTION]) if (TAGS in instagram_post_entity and instagram_post_entity[TAGS] != None): entity[TAGS] = EntityProperty( EdmType.STRING, self.serialize_entity_attribute_value( instagram_post_entity[TAGS])) if (HAS_LOGO in instagram_post_entity and instagram_post_entity[HAS_LOGO] != None): entity[HAS_LOGO] = EntityProperty(EdmType.BOOLEAN, instagram_post_entity[HAS_LOGO]) if (ACCURACY in instagram_post_entity and instagram_post_entity[ACCURACY] != None): entity[ACCURACY] = EntityProperty(EdmType.DOUBLE, instagram_post_entity[ACCURACY]) if (IMAGE_CONTEXT in instagram_post_entity and instagram_post_entity[IMAGE_CONTEXT] != None): entity[IMAGE_CONTEXT] = EntityProperty( EdmType.STRING, self.serialize_entity_attribute_value( instagram_post_entity[IMAGE_CONTEXT])) if (IMAGE_PATH in instagram_post_entity and instagram_post_entity[IMAGE_PATH] != None): entity[IMAGE_PATH] = EntityProperty( EdmType.STRING, instagram_post_entity[IMAGE_PATH]) if (DIMENSIONS in instagram_post_entity and instagram_post_entity[DIMENSIONS] != None): entity[DIMENSIONS] = EntityProperty( EdmType.STRING, self.serialize_entity_attribute_value( instagram_post_entity[DIMENSIONS])) return entity def serialize_entity_attribute_value(self, attribute_value): if (attribute_value == None): return "" return json.dumps(attribute_value, indent=4, ensure_ascii=False) def get_all_entries(self, table_name): return list(self.service.query_entities(table_name))
# blob services. # # See http://go.microsoft.com/fwlink/?linkid=246933 for Storage documentation. # STORAGE_ACCOUNT_NAME = '__paste_your_storage_account_name_here__' STORAGE_ACCOUNT_KEY = '__paste_your_storage_key_here__' if os.environ.get('EMULATED', '').lower() == 'true': # Running in the emulator, so use the development storage account storage_account = CloudStorageAccount(None, None) else: storage_account = CloudStorageAccount(STORAGE_ACCOUNT_NAME, STORAGE_ACCOUNT_KEY) blob_service = storage_account.create_blob_service() table_service = storage_account.create_table_service() queue_service = storage_account.create_queue_service() # # Service Bus is a messaging solution for applications. It sits between # components of your applications and enables them to exchange messages in a # loosely coupled way for improved scale and resiliency. # # See http://go.microsoft.com/fwlink/?linkid=246934 for Service Bus documentation. # SERVICE_BUS_NAMESPACE = '__paste_your_service_bus_namespace_here__' SERVICE_BUS_KEY = '__paste_your_service_bus_key_here__' bus_service = ServiceBusService(SERVICE_BUS_NAMESPACE, SERVICE_BUS_KEY, issuer='owner')
class StorageAccountTest(StorageTestCase): def setUp(self): super(StorageAccountTest, self).setUp() self.account_name = self.settings.STORAGE_ACCOUNT_NAME self.account_key = self.settings.STORAGE_ACCOUNT_KEY self.sas_token = '?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D' self.account = CloudStorageAccount(self.account_name, self.account_key) #--Helpers----------------------------------------------------------------- def validate_service(self, service, type): self.assertIsNotNone(service) self.assertIsInstance(service, type) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) #--Test cases -------------------------------------------------------- def test_create_block_blob_service(self): # Arrange # Act service = self.account.create_block_blob_service() # Assert self.validate_service(service, BlockBlobService) def test_create_page_blob_service(self): # Arrange # Act service = self.account.create_page_blob_service() # Assert self.validate_service(service, PageBlobService) def test_create_append_blob_service(self): # Arrange # Act service = self.account.create_append_blob_service() # Assert self.validate_service(service, AppendBlobService) def test_create_table_service(self): # Arrange # Act service = self.account.create_table_service() # Assert self.validate_service(service, TableService) def test_create_queue_service(self): # Arrange # Act service = self.account.create_queue_service() # Assert self.validate_service(service, QueueService) def test_create_file_service(self): # Arrange # Act service = self.account.create_file_service() # Assert self.validate_service(service, FileService) def test_create_service_no_key(self): # Arrange # Act bad_account = CloudStorageAccount('', '') with self.assertRaises(ValueError): service = bad_account.create_block_blob_service() # Assert def test_create_account_sas(self): # Arrange # Act sas_account = CloudStorageAccount(self.account_name, sas_token=self.sas_token) service = sas_account.create_block_blob_service() # Assert self.assertIsNotNone(service) self.assertEqual(service.account_name, self.account_name) self.assertIsNone(service.account_key) self.assertEqual(service.sas_token, self.sas_token) def test_create_account_sas_and_key(self): # Arrange # Act account = CloudStorageAccount(self.account_name, self.account_key, self.sas_token) service = account.create_block_blob_service() # Assert self.validate_service(service, BlockBlobService) def test_create_account_emulated(self): # Arrange # Act account = CloudStorageAccount(is_emulated=True) service = account.create_block_blob_service() # Assert self.assertIsNotNone(service) self.assertEqual(service.account_name, 'devstoreaccount1') self.assertIsNotNone(service.account_key) @record def test_generate_account_sas(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recordingfile(self.test_mode): return # Arrange token = self.account.generate_shared_access_signature( Services.BLOB, ResourceTypes.OBJECT, AccountPermissions.READ, datetime.utcnow() + timedelta(hours=1), ) service = self.account.create_block_blob_service() data = b'shared access signature with read permission on blob' container_name='container1' blob_name = 'blob1.txt' try: service.create_container(container_name) service.create_blob_from_bytes(container_name, blob_name, data) # Act url = service.make_blob_url( container_name, blob_name, sas_token=token, ) response = requests.get(url) # Assert self.assertTrue(response.ok) self.assertEqual(data, response.content) finally: service.delete_container(container_name)
class StorageTableContext(): """Initializes the repository with the specified settings dict. Required settings in config dict are: - AZURE_STORAGE_NAME - AZURE_STORAGE_KEY - AZURE_KEY_IDENTIFIER - AZURE_SECRET_KEY - AZURE_STORAGE_IS_EMULATED """ _account = None _account_name = '' _account_key = '' _is_emulated = False _kek = None _key_resolver = None _modeldefinitions = [] REQUIRED = True # decorators def getmodeldefinition(self, storageobject, required=False): """ find modeldefinition for StorageTableModel or StorageTableQuery """ if isinstance(storageobject, StorageTableModel): definitionlist = [ definition for definition in self._modeldefinitions if definition['modelname'] == storageobject.__class__.__name__ ] elif isinstance(storageobject, StorageTableQuery): """ StorageTableQuery """ storagemodel = storageobject._storagemodel definitionlist = [ definition for definition in self._modeldefinitions if definition['modelname'] == storagemodel.__class__.__name__ ] else: raise Exception( "Argument is not an StorageTableModel nor an StorageTableQuery" ) # is there only one modeldefinition ? # hopefully! modeldefinition = None if len(definitionlist) == 1: modeldefinition = definitionlist[0] elif len(definitionlist) > 1: raise ModelRegisteredMoreThanOnceError(storageobject) # is there a modeldefinition if required ? if required and modeldefinition is None: raise ModelNotRegisteredError(storageobject) return modeldefinition # constructor def __init__(self, **kwargs): """ parse kwargs """ self._account_name = kwargs.get('AZURE_STORAGE_NAME', '') self._account_key = kwargs.get('AZURE_STORAGE_KEY', '') self._is_emulated = kwargs.get('AZURE_STORAGE_IS_EMULATED', False) self._key_identifier = kwargs.get('AZURE_KEY_IDENTIFIER', '') self._secret_key = kwargs.get('AZURE_SECRET_KEY', '') """ account init """ if self._is_emulated: self._account = CloudStorageAccount(is_emulated=True) elif self._account_name != '' and self._account_key != '': self._account = CloudStorageAccount(self._account_name, self._account_key) else: raise AzureException """ init table model list """ self._modeldefinitions = [] def __createtable__(self, modeldefinition: dict) -> bool: if (not modeldefinition['tableservice'] is None): try: modeldefinition['tableservice'].create_table( modeldefinition['tablename']) return True except Exception as e: log.error('failed to create {} with error {}'.format( tablename, e)) return False else: return False pass def __deletetable__(self, modeldefinition: dict) -> bool: if (not modeldefinition['tableservice'] is None): try: modeldefinition['tableservice'].delete_table( modeldefinition['tablename']) return True except Exception as e: msg = 'failed to create {} with error {}'.format(tablename, e) raise AzureStorageWrapException(msg=msg) else: return False pass def register_model(self, storagemodel: object): """ set up an Tableservice for an StorageTableModel in your Azure Storage Account Will create the Table if not exist! required Parameter is: - storagemodel: StorageTableModel(Object) """ modeldefinition = self.getmodeldefinition(storagemodel, False) if modeldefinition is None: """ test if queuename already exists """ if [ model for model in self._modeldefinitions if model['tablename'] == storagemodel._tablename ]: raise NameConventionError(storagemodel._tablename) """ test if queuename fits to azure naming rules """ if not test_azurestorage_nameconventions(storagemodel._tablename, 'StorageTableModel'): raise NameConventionError(storagemodel._tablename) """ now register model """ modeldefinition = { 'modelname': storagemodel.__class__.__name__, 'tablename': storagemodel._tablename, 'encrypt': storagemodel._encrypt, 'tableservice': self._account.create_table_service() } if modeldefinition['encrypt']: """ encrypt init """ # Create the KEK used for encryption. # KeyWrapper is the provided sample implementation, but the user may use their own object as long as it implements the interface above. kek = KeyWrapper(self._key_identifier, self._secret_key) # Key identifier # Create the key resolver used for decryption. # KeyResolver is the provided sample implementation, but the user may use whatever implementation they choose so long as the function set on the service object behaves appropriately. key_resolver = KeyResolver() key_resolver.put_key(kek) # Create the EncryptionResolver Function to determine Properties to en/decrypt encryptionresolver = self.__encryptionresolver__( modeldefinition['encrypt']) # Set the require Encryption, KEK and key resolver on the service object. modeldefinition['tableservice'].key_encryption_key = kek modeldefinition[ 'tableservice'].key_resolver_funcion = key_resolver.resolve_key modeldefinition[ 'tableservice'].encryption_resolver_function = encryptionresolver pass self.__createtable__(modeldefinition) self._modeldefinitions.append(modeldefinition) log.info( 'model {} registered successfully. Models are {!s}.'.format( modeldefinition['modelname'], [model['modelname'] for model in self._modeldefinitions])) else: log.info('model {} already registered. Models are {!s}.'.format( modeldefinition['modelname'], [model['modelname'] for model in self._modeldefinitions])) def unregister_model(self, storagemodel: object, delete_table=False): """ clear up an Tableservice for an StorageTableModel in your Azure Storage Account Will delete the Table if delete_table Flag is True! required Parameter is: - storagemodel: StorageTableModel(Object) Optional Parameter is: - delete_table: bool """ # get modeldefinition modeldefinition = self.getmodeldefinition(storagemodel, True) # remove from modeldefinitions for i in range(len(self._modeldefinitions)): if self._modeldefinitions[i]['modelname'] == modeldefinition[ 'modelname']: del self._modeldefinitions[i] break # delete table from storage if delete_table == True if delete_table: self.__deletetable__(modeldefinition) pass # methods def exists(self, storagemodel) -> bool: modeldefinition = self.getmodeldefinition(storagemodel, True) exists = False if storagemodel._exists is None: try: pk = storagemodel.getPartitionKey() rk = storagemodel.getRowKey() entity = modeldefinition['tableservice'].get_entity( modeldefinition['tablename'], pk, rk) storagemodel._exists = True exists = True except AzureMissingResourceHttpError: storagemodel._exists = False except Exception as e: msg = 'failed to test {} with error {}'.format( modeldefinition['tablename'], e) raise AzureStorageWrapException(msg=msg) else: exists = storagemodel._exists return exists def get(self, storagemodel) -> StorageTableModel: """ load entity data from storage to vars in self """ modeldefinition = self.getmodeldefinition(storagemodel, True) try: pk = storagemodel.getPartitionKey() rk = storagemodel.getRowKey() entity = modeldefinition['tableservice'].get_entity( modeldefinition['tablename'], pk, rk) storagemodel._exists = True """ sync with entity values """ for key, default in vars(storagemodel).items(): if not key.startswith('_') and key not in [ '', 'PartitionKey', 'RowKey' ]: value = getattr(entity, key, None) if not value is None: setattr(storagemodel, key, value) except AzureMissingResourceHttpError as e: log.debug( 'can not get table entity: Table {}, PartitionKey {}, RowKey {} because {!s}' .format(modeldefinition['tablename'], pk, rk, e)) storagemodel._exists = False except Exception as e: msg = 'can not get table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format( modeldefinition['tablename'], pk, rk, e) raise AzureStorageWrapException(msg=msg) finally: return storagemodel def insert(self, storagemodel) -> StorageTableModel: """ insert model into storage """ modeldefinition = self.getmodeldefinition(storagemodel, True) try: modeldefinition['tableservice'].insert_or_replace_entity( modeldefinition['tablename'], storagemodel.entity()) storagemodel._exists = True except AzureMissingResourceHttpError as e: storagemodel._exists = False log.debug( 'can not insert or replace table entity: Table {}, PartitionKey {}, RowKey {} because {!s}' .format(modeldefinition['tablename'], storagemodel.getPartitionKey(), storagemodel.getRowKey(), e)) except Exception as e: storagemodel._exists = False msg = 'can not insert or replace table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format( modeldefinition['tablename'], storagemodel.PartitionKey, storagemodel.RowKey, e) raise AzureStorageWrapException(msg=msg) finally: return storagemodel def merge(self, storagemodel) -> StorageTableModel: """ try to merge entry """ modeldefinition = self.getmodeldefinition(storagemodel, True) try: pk = storagemodel.getPartitionKey() rk = storagemodel.getRowKey() entity = modeldefinition['tableservice'].get_entity( modeldefinition['tablename'], pk, rk) """ merge with entity values """ for key, default in vars(storagemodel.__class__).items(): if not key.startswith('_') and key not in ['']: if isinstance(default, PartitionKey) or isinstance( default, RowKey) or isinstance( default, EncryptKey): default = default._default newvalue = getattr(storagemodel, key, None) if (newvalue is None) or (newvalue == default): oldvalue = getattr(entity, key, default) setattr(storagemodel, key, oldvalue) modeldefinition['tableservice'].insert_or_replace_entity( modeldefinition['tablename'], storagemodel.entity()) storagemodel._exists = True except AzureMissingResourceHttpError as e: log.debug( 'can not merge table entity: Table {}, PartitionKey {}, RowKey {} because {!s}' .format(modeldefinition['tablename'], pk, rk, e)) except Exception as e: log.debug( 'can not merge table entity: Table {}, PartitionKey {}, RowKey {} because {!s}' .format(modeldefinition['tablename'], pk, rk, e)) finally: return storagemodel def delete(self, storagemodel): """ delete existing Entity """ modeldefinition = self.getmodeldefinition(storagemodel, True) pk = storagemodel.getPartitionKey() rk = storagemodel.getRowKey() try: modeldefinition['tableservice'].delete_entity( modeldefinition['tablename'], pk, rk) storagemodel._exists = False except AzureMissingResourceHttpError as e: log.debug( 'can not delete table entity: Table {}, PartitionKey {}, RowKey {} because {!s}' .format(modeldefinition['tablename'], pk, rk, e)) finally: return storagemodel def query(self, storagequery) -> StorageTableQuery: modeldefinition = self.getmodeldefinition(storagequery, True) try: if (not storagequery._select is None) and (storagequery._select != ''): storagequery.extend( modeldefinition['tableservice'].query_entities( modeldefinition['tablename'], filter=storagequery._queryfilter, select=storagequery._select)) else: storagequery.extend( modeldefinition['tableservice'].query_entities( modeldefinition['tablename'], filter=storagequery._queryfilter)) except AzureMissingResourceHttpError as e: storagequery = [] log.debug( 'can not query table {} with filters {} because {!s}'.format( modeldefinition['tablename'], storagequery._queryfilter, e)) except Exception as e: msg = 'can not query table {} with filters {} because {!s}'.format( modeldefinition['tablename'], storagequery._queryfilter, e) raise AzureStorageWrapException(msg=msg) return storagequery def table_isempty(self, tablename, PartitionKey='', RowKey='') -> bool: if (not self._tableservice is None): filter = "PartitionKey eq '{}'".format( PartitionKey) if PartitionKey != '' else '' if filter == '': filter = "RowKey eq '{}'".format( RowKey) if RowKey != '' else '' else: filter = filter + ("and RowKey eq '{}'".format(RowKey) if RowKey != '' else '') try: entities = list(modeldefinition['tableservice'].query_entities( tablename, filter=filter, select='PartitionKey', num_results=1)) if len(entities) == 1: return False else: return True except AzureMissingResourceHttpError as e: log.debug('failed to query {} with error {}'.format( tablename, e)) return True except Exception as e: msg = '{!s}'.format(e) raise AzureStorageWrapException(msg=msg) else: return True pass def __encryptionresolver__(self, encryptproperties): def encryptionresolver(pk, rk, property_name): if property_name in encryptproperties: return True else: return False return encryptionresolver
def Init(name,key): global account, table_service account = CloudStorageAccount(name,key) print("storage account created") table_service = account.create_table_service() print("table service created")
self.muscleTargeted = muscleTargeted self.url = url baseUrl = "https://www.bodybuilding.com" url = baseUrl + "/exercises/finder" page = 1 workToDo = True exercises = [] batch = TableBatch() account = CloudStorageAccount(is_emulated=True) table_service = account.create_table_service() table_service.create_table('exercises') counter = 0 while (workToDo): headers = {"x-bb-ex-skip-finder-form": "YES"} urlToGet = url + "/" + str(page) print(urlToGet) r = requests.get(urlToGet, headers=headers) soup = BeautifulSoup(r.text, "html.parser") exerciseDivs = soup.findAll("div", {"class": "ExResult-row"})