def setUp(self): super(StorageGetFileTest, self).setUp() # test chunking functionality by reducing the threshold # for chunking and the size of each chunk, otherwise # the tests would take too long to execute self.MAX_SINGLE_GET_SIZE = 32 * 1024 self.MAX_CHUNK_GET_SIZE = 4 * 1024 url = self.get_file_url() credential = self.get_shared_key_credential() self.fsc = ShareServiceClient( url, credential=credential, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) self.share_name = self.get_resource_name('utshare') self.directory_name = self.get_resource_name('utdir') if not self.is_playback(): share = self.fsc.create_share(self.share_name) share.create_directory(self.directory_name) self.byte_file = self.get_resource_name('bytefile') self.byte_data = self.get_random_bytes(64 * 1024 + 5) if not self.is_playback(): byte_file = self.directory_name + '/' + self.byte_file file_client = ShareFileClient(self.get_file_url(), share_name=self.share_name, file_path=byte_file, credential=credential) file_client.upload_file(self.byte_data)
def test_old_api_copy_file_succeeds(self, resource_group, location, storage_account, storage_account_key): fsc = ShareServiceClient(self.account_url(storage_account, "file"), credential=storage_account_key, max_range_size=4 * 1024, api_version=self.api_version_1) share = self._create_share(fsc) file_name = self._get_file_reference() source_client = share.get_file_client(file_name) source_client.upload_file(self.short_byte_data) source_prop = source_client.get_file_properties() file_client = ShareFileClient(self.account_url(storage_account, "file"), share_name=share.share_name, file_path='file1copy', credential=storage_account_key, api_version=self.api_version_1) # Act copy = file_client.start_copy_from_url(source_client.url) # Assert dest_prop = file_client.get_file_properties() # to make sure the acl is copied from source self.assertEqual(source_prop['permission_key'], dest_prop['permission_key']) self.assertIsNotNone(copy) self.assertEqual(copy['copy_status'], 'success') self.assertIsNotNone(copy['copy_id']) copy_file = file_client.download_file().readall() self.assertEqual(copy_file, self.short_byte_data)
def test_ranged_get_file_to_path_invalid_range_non_parallel(self): # Arrange file_size = 1024 file_data = self.get_random_bytes(file_size) file_name = self._get_file_reference() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + file_name, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_client.upload_file(file_data) # Act start = 3 end_range = 2 * self.MAX_SINGLE_GET_SIZE with open(FILE_PATH, 'wb') as stream: length = end_range - start + 1 bytes_read = file_client.download_file( offset=start, length=length, max_concurrency=1).readinto(stream) # Assert self.assertIsInstance(bytes_read, int) with open(FILE_PATH, 'rb') as stream: actual = stream.read() self.assertEqual(file_data[start:file_size], actual)
def test_get_file_to_stream_with_progress(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) progress = [] def callback(response): current = response.context['download_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) # Act with open(FILE_PATH, 'wb') as stream: bytes_read = file_client.download_file( raw_response_hook=callback, max_concurrency=2).readinto(stream) # Assert self.assertIsInstance(bytes_read, int) with open(FILE_PATH, 'rb') as stream: actual = stream.read() self.assertEqual(self.byte_data, actual) self.assert_download_progress(len(self.byte_data), self.MAX_CHUNK_GET_SIZE, self.MAX_SINGLE_GET_SIZE, progress)
def test_get_file_to_text_with_progress(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange text_file = self.get_resource_name('textfile') text_data = self.get_random_text_data(self.MAX_SINGLE_GET_SIZE + 1) file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + text_file, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_client.upload_file(text_data) progress = [] def callback(response): current = response.context['download_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) # Act file_content = file_client.download_file(raw_response_hook=callback, max_concurrency=2, encoding='utf-8').readall() # Assert self.assertEqual(text_data, file_content) self.assert_download_progress(len(text_data.encode('utf-8')), self.MAX_CHUNK_GET_SIZE, self.MAX_SINGLE_GET_SIZE, progress)
def test_get_file_to_text_non_parallel(self): # Arrange text_file = self._get_file_reference() text_data = self.get_random_text_data(self.MAX_SINGLE_GET_SIZE + 1) file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + text_file, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_client.upload_file(text_data) progress = [] def callback(response): current = response.context['download_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) # Act file_content = file_client.download_file(raw_response_hook=callback, max_concurrency=1, encoding='utf-8').readall() # Assert self.assertEqual(text_data, file_content) self.assert_download_progress(len(text_data), self.MAX_CHUNK_GET_SIZE, self.MAX_SINGLE_GET_SIZE, progress)
def test_get_file_to_stream_small(self): # Arrange file_data = self.get_random_bytes(1024) file_name = self._get_file_reference() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + file_name, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_client.upload_file(file_data) progress = [] def callback(response): current = response.context['download_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) # Act with open(FILE_PATH, 'wb') as stream: bytes_read = file_client.download_file( raw_response_hook=callback, max_concurrency=1).readinto(stream) # Assert self.assertIsInstance(bytes_read, int) with open(FILE_PATH, 'rb') as stream: actual = stream.read() self.assertEqual(file_data, actual) self.assert_download_progress(len(file_data), self.MAX_CHUNK_GET_SIZE, self.MAX_SINGLE_GET_SIZE, progress)
def test_get_file_to_text_with_encoding_and_progress(self): # Arrange text = u'hello 啊齄丂狛狜 world' data = text.encode('utf-16') file_name = self._get_file_reference() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + file_name, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_client.upload_file(data) # Act progress = [] def callback(response): current = response.context['download_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) file_content = file_client.download_file(raw_response_hook=callback, encoding='UTF-16').readall() # Assert self.assertEqual(text, file_content) self.assert_download_progress(len(data), self.MAX_CHUNK_GET_SIZE, self.MAX_SINGLE_GET_SIZE, progress)
def test_ranged_get_file_to_path_with_single_byte(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) # Act end_range = self.MAX_SINGLE_GET_SIZE + 1024 with open(FILE_PATH, 'wb') as stream: bytes_read = file_client.download_file(offset=0, length=1).readinto(stream) # Assert self.assertIsInstance(bytes_read, int) with open(FILE_PATH, 'rb') as stream: actual = stream.read() self.assertEqual(1, len(actual)) self.assertEqual(self.byte_data[0], actual[0])
def exist_az_file(file_client: ShareFileClient) -> bool: """Check if existed Azure File. :param file_client: :type file_client: ShareClient :return: True if Azure file is existed. """ try: file_client.get_file_properties() return True except ResourceNotFoundError: return False
def test_get_file_non_seekable_parallel_from_snapshot(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange # Create a snapshot of the share and delete the file share_client = self.fsc.get_share_client(self.share_name) share_snapshot = share_client.create_snapshot() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, credential=self.settings.STORAGE_ACCOUNT_KEY) file_client.delete_file() snapshot_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, snapshot=share_snapshot, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) # Act with open(FILE_PATH, 'wb') as stream: non_seekable_stream = StorageGetFileTest.NonSeekableFile(stream) with self.assertRaises(ValueError): snapshot_client.download_file( max_concurrency=2).readinto(non_seekable_stream)
def __init__(self, connection_string, share_name, file_path): self.share_cli = ShareClient.from_connection_string( conn_str=connection_string, share_name=share_name) self.file_cli = ShareFileClient.from_connection_string( conn_str=connection_string, share_name=share_name, file_path=file_path)
def download_file(self, azure_storage_path_to_file: str): if settings.ENVIRONMENT == "prod": output_filename = "temp_file.sgy" elif settings.ENVIRONMENT == "dev": output_filename = "data/temp_file.sgy" # Create a ShareFileClient from a connection string file_client = ShareFileClient.from_connection_string( self.conn_str, self.share, azure_storage_path_to_file) file_properties = file_client.get_file_properties() file_size = file_properties.content_length # size of file in bytes if file_size > settings.FILE_SIZE_LIMIT_IN_BYTES: raise DownloadFileException( "Tried to decimate a file larger than 400 MB. Abort.") logger.info( f"Downloading file from azure storage to local file {output_filename}" ) # Open a file for writing bytes on the local system - will write over existing file with open(output_filename, "wb") as data: # Download the file from Azure into a stream stream = file_client.download_file() # Write the stream to the local file data.write(stream.readall()) return output_filename
def test_invalid_api_version(self): with pytest.raises(ValueError) as error: ShareServiceClient("https://foo.file.core.windows.net/account", credential="fake_key", api_version="foo") self.assertTrue( str(error.value).startswith("Unsupported API version 'foo'.")) with pytest.raises(ValueError) as error: ShareClient("https://foo.file.core.windows.net/account", "share_name", credential="fake_key", api_version="foo") self.assertTrue( str(error.value).startswith("Unsupported API version 'foo'.")) with pytest.raises(ValueError) as error: ShareDirectoryClient("https://foo.file.core.windows.net/account", "share_name", "dir_path", credential="fake_key", api_version="foo") self.assertTrue( str(error.value).startswith("Unsupported API version 'foo'.")) with pytest.raises(ValueError) as error: ShareFileClient("https://foo.file.core.windows.net/account", "share", self._get_file_reference(), credential="fake_key", api_version="foo") self.assertTrue( str(error.value).startswith("Unsupported API version 'foo'."))
def __init__(self, conn_str: str, share_name: str, file_path: str) -> None: """Initialize the sender class. Retrieves the DB from the file share. All the parameters of __init__ are there to retrieve the DB. Parameters ---------- conn_str Connection strin to the storage account containing the DB. Every Function App has an storage account associated with it. It's connection strin is stored in the default env variable AzureWebJobsStorage. share_name Name of the share where the DB is kept. file_path Path within the File Share to the DB. """ file_client = ShareFileClient.from_connection_string( conn_str=conn_str, share_name=share_name, file_path=file_path, ) data = file_client.download_file() self.email_db = json.loads(data.readall())
def helper_download_dir(source_dir, desti_dir, c_str, s_name, space=""): dir_client = ShareDirectoryClient.from_connection_string( conn_str=c_str, share_name=s_name, directory_path=source_dir) my_list = [] for item in dir_client.list_directories_and_files(): my_list.append(item) for ele in my_list: print(space, ele) if ele['is_directory']: os.mkdir(desti_dir + "/" + ele['name']) helper_download_dir(source_dir + "/" + ele['name'], desti_dir + "/" + ele['name'], c_str, s_name, space + " ") else: file_client = ShareFileClient.from_connection_string( conn_str=c_str, share_name=s_name, file_path=source_dir + "/" + ele['name']) with open(desti_dir + "/" + ele['name'], "wb") as data: stream = file_client.download_file() data.write(stream.readall())
def delete_dir_tree(c_str, s_name, d_name, space=""): dir_client = ShareDirectoryClient.from_connection_string( conn_str=c_str, share_name=s_name, directory_path=d_name) my_list = [] for item in dir_client.list_directories_and_files(): my_list.append(item) for ele in my_list: print(space, ele) if ele['is_directory']: delete_dir_tree(c_str, s_name, d_name + "/" + ele['name'], space=space + " ") else: file_client = ShareFileClient.from_connection_string( conn_str=c_str, share_name=s_name, file_path=d_name + "/" + ele['name']) file_client.delete_file() dir_client.delete_directory()
def upload_a_file_to_share(self): # Instantiate the ShareClient from a connection string from azure.storage.fileshare import ShareClient share = ShareClient.from_connection_string(self.connection_string, share_name="helloworld2") # Create the share share.create_share() try: # Instantiate the ShareFileClient from a connection string # [START create_file_client] from azure.storage.fileshare import ShareFileClient file = ShareFileClient.from_connection_string( self.connection_string, share_name="helloworld2", file_path="myfile") # [END create_file_client] # Upload a file with open(SOURCE_FILE, "rb") as source_file: file.upload_file(source_file) finally: # Delete the share share.delete_share()
def download_snapshot_file(self, connection_string, share_name, snapshot_time, dir_name, file_name): try: # Build the remote path source_file_path = dir_name + "/" + file_name # Add a prefix to the local filename to # indicate it's a file from a snapshot dest_file_name = "SNAPSHOT-" + file_name # Create a ShareFileClient from a connection string snapshot_file_client = ShareFileClient.from_connection_string( conn_str=connection_string, share_name=share_name, file_path=source_file_path, snapshot=snapshot_time) print("Downloading to:", dest_file_name) # Open a file for writing bytes on the local system with open(dest_file_name, "wb") as data: # Download the file from Azure into a stream stream = snapshot_file_client.download_file() # Write the stream to the local file data.write(stream.readall()) except ResourceNotFoundError as ex: print("ResourceNotFoundError:", ex.message)
def downloadFile(ipcSn, dirName, fileName, destDir): file_path=ipcSn + '/'+dirName+'/'+fileName destDir = destDir + '/' + fileName print("downloading {}: {} {} {}".format(destDir, ipcSn, dirName, file_path)) with ShareFileClient.from_connection_string(conn_str=CONNSTR, share_name=SHARENAME, file_path=file_path) as fc: with open(destDir, "wb") as f: data = fc.download_file() data.readinto(f)
def test_file_client_api_version_property(self): file_client = ShareFileClient( "https://foo.file.core.windows.net/account", "share", self._get_file_reference(), credential="fake_key") self.assertEqual(file_client.api_version, self.api_version_2) self.assertEqual(file_client._client._config.version, self.api_version_2) file_client = ShareFileClient( "https://foo.file.core.windows.net/account", "share", self._get_file_reference(), credential="fake_key", api_version=self.api_version_1) self.assertEqual(file_client.api_version, self.api_version_1) self.assertEqual(file_client._client._config.version, self.api_version_1)
def test_get_file_non_seekable_from_snapshot(self): # Arrange # Create a snapshot of the share and delete the file share_client = self.fsc.get_share_client(self.share_name) share_snapshot = share_client.create_snapshot() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, credential=self.settings.STORAGE_ACCOUNT_KEY) file_client.delete_file() snapshot_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, snapshot=share_snapshot, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) # Act with open(FILE_PATH, 'wb') as stream: non_seekable_stream = StorageGetFileTest.NonSeekableFile(stream) bytes_read = snapshot_client.download_file( max_concurrency=1).readinto(non_seekable_stream) # Assert self.assertIsInstance(bytes_read, int) with open(FILE_PATH, 'rb') as stream: actual = stream.read() self.assertEqual(self.byte_data, actual)
def test_get_file_range_with_md5(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_content = file_client.download_file(offset=0, length=1024, validate_content=True) # Assert self.assertIsNone(file_content.properties.content_settings.content_md5) # Arrange props = file_client.get_file_properties() props.content_settings.content_md5 = b'MDAwMDAwMDA=' file_client.set_http_headers(props.content_settings) # Act file_content = file_client.download_file(offset=0, length=1024, validate_content=True) # Assert self.assertEqual(b'MDAwMDAwMDA=', file_content.properties.content_settings.content_md5)
def test_get_file_to_stream_from_snapshot(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange # Create a snapshot of the share and delete the file share_client = self.fsc.get_share_client(self.share_name) share_snapshot = share_client.create_snapshot() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, credential=self.settings.STORAGE_ACCOUNT_KEY) file_client.delete_file() snapshot_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, snapshot=share_snapshot, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) # Act with open(FILE_PATH, 'wb') as stream: bytes_read = snapshot_client.download_file( max_concurrency=2).readinto(stream) # Assert self.assertIsInstance(bytes_read, int) with open(FILE_PATH, 'rb') as stream: actual = stream.read() self.assertEqual(self.byte_data, actual)
def file_exists(self, path: str) -> bool: try: file_client = ShareFileClient.from_connection_string( self.conn_str, self.share, path) # If getting props does not raise an error, we assume the file exists file_client.get_file_properties() return True except ResourceNotFoundError: return False
def test_unicode_get_file_unicode_data(self): # Arrange file_data = u'hello world啊齄丂狛狜'.encode('utf-8') file_name = self._get_file_reference() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + file_name, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_client.upload_file(file_data) # Act file_content = file_client.download_file().readall() # Assert self.assertEqual(file_content, file_data)
def test_get_file_properties_server_encryption(self): # Arrange file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) # Act props = file_client.get_file_properties() # Assert if self.is_file_encryption_enabled(): self.assertTrue(props.server_encrypted) else: self.assertFalse(props.server_encrypted)
def test_get_file_to_text_with_encoding(self): # Arrange text = u'hello 啊齄丂狛狜 world' data = text.encode('utf-16') file_name = self._get_file_reference() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + file_name, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_client.upload_file(data) # Act file_content = file_client.download_file(encoding='UTF-16').readall() # Assert self.assertEqual(text, file_content)
def test_get_file_no_content(self): # Arrange file_data = b'' file_name = self._get_file_reference() file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + file_name, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) file_client.upload_file(file_data) # Act file_output = file_client.download_file() # Assert self.assertEqual(file_data, file_output.readall()) self.assertEqual(0, file_output.properties.size)
def test_get_file_to_bytes(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange file_client = ShareFileClient( self.get_file_url(), share_name=self.share_name, file_path=self.directory_name + '/' + self.byte_file, credential=self.settings.STORAGE_ACCOUNT_KEY, max_single_get_size=self.MAX_SINGLE_GET_SIZE, max_chunk_get_size=self.MAX_CHUNK_GET_SIZE) # Act file_content = file_client.download_file(max_concurrency=2).readall() # Assert self.assertEqual(self.byte_data, file_content)