def test_rename_directory_share_sas(self, storage_account_name,
storage_account_key):
self._setup(storage_account_name, storage_account_key)
share_client = self.fsc.get_share_client(self.share_name)
token = generate_share_sas(share_client.account_name,
share_client.share_name,
share_client.credential.account_key,
expiry=datetime.utcnow() +
timedelta(hours=1),
permission=ShareSasPermissions(read=True,
write=True))
source_directory = ShareDirectoryClient(self.account_url(
storage_account_name, 'file'),
share_client.share_name,
'dir1',
credential=token)
source_directory.create_directory()
# Act
new_directory = source_directory.rename_directory('dir2' + '?' + token)
# Assert
props = new_directory.get_directory_properties()
self.assertIsNotNone(props)
def recursive_download(
client: ShareDirectoryClient, destination: str, max_concurrency: int, starts_with: str = ""
) -> None:
"""
Helper function for `download`. Recursively downloads remote fileshare directory locally
"""
try:
items = list(client.list_directories_and_files(name_starts_with=starts_with))
files = [item for item in items if not item["is_directory"]]
folders = [item for item in items if item["is_directory"]]
for f in files:
Path(destination).mkdir(parents=True, exist_ok=True)
file_name = f["name"]
file_client = client.get_file_client(file_name)
file_content = file_client.download_file(max_concurrency=max_concurrency)
local_path = Path(destination, file_name)
with open(local_path, "wb") as file_data:
file_data.write(file_content.readall())
for f in folders:
sub_client = client.get_subdirectory_client(f["name"])
destination = "/".join((destination, f["name"]))
recursive_download(sub_client, destination=destination, max_concurrency=max_concurrency)
except Exception:
raise Exception(f"Saving fileshare directory with prefix {starts_with} was unsuccessful.")
def test_invalid_api_version(self):
with pytest.raises(ValueError) as error:
ShareServiceClient("https://foo.file.core.windows.net/account",
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
ShareClient("https://foo.file.core.windows.net/account",
"share_name",
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
ShareDirectoryClient("https://foo.file.core.windows.net/account",
"share_name",
"dir_path",
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
ShareFileClient("https://foo.file.core.windows.net/account",
"share",
self._get_file_reference(),
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
def helper_download_dir(source_dir, desti_dir, c_str, s_name, space=""):
dir_client = ShareDirectoryClient.from_connection_string(
conn_str=c_str, share_name=s_name, directory_path=source_dir)
my_list = []
for item in dir_client.list_directories_and_files():
my_list.append(item)
for ele in my_list:
print(space, ele)
if ele['is_directory']:
os.mkdir(desti_dir + "/" + ele['name'])
helper_download_dir(source_dir + "/" + ele['name'],
desti_dir + "/" + ele['name'], c_str, s_name,
space + " ")
else:
file_client = ShareFileClient.from_connection_string(
conn_str=c_str,
share_name=s_name,
file_path=source_dir + "/" + ele['name'])
with open(desti_dir + "/" + ele['name'], "wb") as data:
stream = file_client.download_file()
data.write(stream.readall())
def delete_dir_tree(c_str, s_name, d_name, space=""):
dir_client = ShareDirectoryClient.from_connection_string(
conn_str=c_str, share_name=s_name, directory_path=d_name)
my_list = []
for item in dir_client.list_directories_and_files():
my_list.append(item)
for ele in my_list:
print(space, ele)
if ele['is_directory']:
delete_dir_tree(c_str,
s_name,
d_name + "/" + ele['name'],
space=space + " ")
else:
file_client = ShareFileClient.from_connection_string(
conn_str=c_str,
share_name=s_name,
file_path=d_name + "/" + ele['name'])
file_client.delete_file()
dir_client.delete_directory()
def create_directory(self, directory: str) -> ShareDirectoryClient:
dir_client = ShareDirectoryClient.from_connection_string(
self.conn_str, self.share, directory)
try:
dir_client.create_directory()
except ResourceExistsError:
logger.debug(f"Directory '{directory}' already exists")
return dir_client
def __init__(self, credential: str, file_share_name: str, account_url: str):
self.directory_client = ShareDirectoryClient(
account_url=account_url,
credential=credential,
share_name=file_share_name,
directory_path=AZ_ML_ARTIFACT_DIRECTORY,
)
self.file_share = file_share_name
self.total_file_count = 1
self.uploaded_file_count = 0
try:
self.directory_client.create_directory()
except ResourceExistsError:
pass
self.subdirectory_client = None
def create_directory(self, connection_string, share_name, dir_name):
try:
# Create a ShareDirectoryClient from a connection string
dir_client = ShareDirectoryClient.from_connection_string(
connection_string, share_name, dir_name)
print("Creating directory:", share_name + "/" + dir_name)
dir_client.create_directory()
except ResourceExistsError as ex:
print("ResourceExistsError:", ex.message)
def test_directory_client_api_version_property(self):
dir_client = ShareDirectoryClient(
"https://foo.file.core.windows.net/account",
"share_name",
"dir_path",
credential="fake_key")
self.assertEqual(dir_client.api_version, self.api_version_2)
self.assertEqual(dir_client._client._config.version,
self.api_version_2)
dir_client = ShareDirectoryClient(
"https://foo.file.core.windows.net/account",
"share_name",
"dir_path",
credential="fake_key",
api_version=self.api_version_1)
self.assertEqual(dir_client.api_version, self.api_version_1)
self.assertEqual(dir_client._client._config.version,
self.api_version_1)
subdir_client = dir_client.get_subdirectory_client("foo")
self.assertEqual(subdir_client.api_version, self.api_version_1)
self.assertEqual(subdir_client._client._config.version,
self.api_version_1)
file_client = dir_client.get_file_client("foo")
self.assertEqual(file_client.api_version, self.api_version_1)
self.assertEqual(file_client._client._config.version,
self.api_version_1)
def create_repo_structure(d_name, sub_dirs):
filepathrepo = "configuration/repo_config.json"
c_str, s_name = read_json_repo(filepathrepo)
dir_client = ShareDirectoryClient.from_connection_string(c_str, s_name, d_name)
print(d_name)
dir_client.create_directory()
for subd_name in sub_dirs:
print(" ", subd_name)
dir_client.create_subdirectory(subd_name)
def upload_source(source_name, source_dir, desti_dir, c_str, s_name, useless_ele = {"__pycache__"}, space = ""):
if os.path.isdir(source_dir + "/" + source_name):
dir_client = ShareDirectoryClient.from_connection_string(conn_str=c_str, share_name=s_name, directory_path=desti_dir + "/" + source_name)
dir_client.create_directory()
print(source_dir + "/" + source_name)
helper_copy_dir(source_dir + "/" + source_name, desti_dir + "/" + source_name, c_str, s_name, useless_ele, space = space)
else:
file_client = ShareFileClient.from_connection_string(conn_str=c_str, share_name=s_name, file_path=desti_dir + "/" + source_name)
with open(source_dir + "/" + source_name, "rb") as source_file:
file_client.upload_file(source_file)
print("Upload Complete")
def main(q):
rootc = ShareDirectoryClient.from_connection_string(conn_str=CONNSTR,
share_name=SHARENAME,
directory_path=".")
rootd = rootc.list_directories_and_files()
# download files
fileCnt = 0
totalSize = 0
totalTime = 0
bBreak = False
for ipc in rootd:
if bBreak: break
if 'is_directory' in ipc and ipc['is_directory'] and (
'-' not in ipc["name"]):
ipcSn = ipc["name"]
ipcc = rootc.get_subdirectory_client(ipc["name"])
ipcd = ipcc.list_directories_and_files()
for d in ipcd:
if bBreak: break
if 'is_directory' in d and d['is_directory']:
dirName = d["name"]
vf = ipcc.get_subdirectory_client(
d["name"]).list_directories_and_files()
for e in vf:
if bBreak: break
m = re.match(r"(\d{13})-(\d{13}).mp4", e["name"])
if m:
fileName = e["name"]
fileSize = e["size"]
ts = int(int(m.group(1)) / 1000)
te = int(int(m.group(2)) / 1000)
delta = te - ts
print("{} {} {:.3f}MB".format(
ipcSn, fileName, fileSize / 1024 /
1024)) #datetime.datetime.fromtimestamp(te))
fileCnt += 1
if fileCnt >= 3:
bBreak = True
now = datetime.datetime.now()
tasks.append(
asyncio.ensure_future(downloadFile(
ipcSn, dirName, fileName, videoQue),
loop=event_loop))
event_loop.run_until_complete(asyncio.gather(*tasks))
def helper_copy_dir(source_dir, desti_dir, c_str, s_name, useless_ele, space = ""):
for ele in os.listdir(source_dir):
if ele in useless_ele:
continue
print(space, int(os.path.isdir(source_dir + "/" + ele)), ele)
if os.path.isdir(source_dir + "/" + ele):
dir_client = ShareDirectoryClient.from_connection_string(conn_str=c_str, share_name=s_name, directory_path=desti_dir + "/" + ele)
dir_client.create_directory()
helper_copy_dir(source_dir + "/" + ele, desti_dir + "/" + ele, c_str, s_name, useless_ele, space = space + " ")
else:
file_client = ShareFileClient.from_connection_string(conn_str=c_str, share_name=s_name, file_path=desti_dir + "/" + ele)
with open(source_dir + "/" + ele, "rb") as source_file:
file_client.upload_file(source_file)
def list_files_in_azure_directory(connection_string, share_name, dir_path):
try:
files = []
parent_dir = ShareDirectoryClient.from_connection_string(
conn_str=connection_string,
share_name=share_name,
directory_path=dir_path)
for item in list(parent_dir.list_directories_and_files()):
if item['is_directory']:
files += (list_files_in_azure_directory(
connection_string, share_name,
os.path.join(dir_path, item['name'])))
else:
files.append(os.path.join(dir_path, item['name']))
return files
except ResourceNotFoundError as ex:
print("ResourceNotFoundError:", ex.message)
def download_source(source_name,
source_dir,
desti_dir,
c_str,
s_name,
space=""):
dir_client = ShareDirectoryClient.from_connection_string(
conn_str=c_str, share_name=s_name, directory_path=source_dir)
flag = True
sorce_info = None
for ele in dir_client.list_directories_and_files():
if ele['name'] == source_name:
sorce_info = ele
flag = False
break
if flag:
print("source Not Exist")
return
print(sorce_info)
if sorce_info['is_directory']:
os.mkdir(desti_dir + "/" + ele['name'])
helper_download_dir(source_dir + "/" + ele['name'],
desti_dir + "/" + ele['name'], c_str, s_name,
space + " ")
else:
file_client = ShareFileClient.from_connection_string(
conn_str=c_str,
share_name=s_name,
file_path=source_dir + "/" + ele['name'])
with open(desti_dir + "/" + ele['name'], "wb") as data:
stream = file_client.download_file()
data.write(stream.readall())
print("Download Complete")
def download(connection_string, share_name, source_dir_path, local_image_path):
try:
files = []
parent_dir = ShareDirectoryClient.from_connection_string(conn_str=connection_string, share_name=share_name,
directory_path=source_dir_path)
for item in list(parent_dir.list_directories_and_files()):
if item['is_directory']:
os.makedirs(os.path.join(local_image_path, item['name']))
download(
connection_string,
share_name,
os.path.join(source_dir_path, item['name']),
os.path.join(local_image_path, item['name']))
else:
source_file_path = os.path.join(source_dir_path, item['name'])
download_from_azure_file_share(connection_string, share_name, source_file_path, local_image_path)
return files
except ResourceNotFoundError as ex:
print("ResourceNotFoundError:", ex.message)
def test_create_service_with_custom_account_endpoint_path(self):
custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token
for service_type in SERVICES.items():
conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};FileEndpoint={};'.format(
self.account_name, self.account_key, custom_account_url)
# Act
service = service_type[0].from_connection_string(
conn_string, share_name="foo", directory_path="bar", file_path="baz")
# Assert
self.assertEqual(service.account_name, self.account_name)
self.assertEqual(service.credential.account_name, self.account_name)
self.assertEqual(service.credential.account_key, self.account_key)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
service = ShareServiceClient(account_url=custom_account_url)
self.assertEqual(service.account_name, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/?'))
service = ShareClient(account_url=custom_account_url, share_name="foo", snapshot="snap")
self.assertEqual(service.account_name, None)
self.assertEqual(service.share_name, "foo")
self.assertEqual(service.snapshot, "snap")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?sharesnapshot=snap&'))
service = ShareDirectoryClient(account_url=custom_account_url, share_name='foo', directory_path="bar/baz", snapshot="snap")
self.assertEqual(service.account_name, None)
self.assertEqual(service.share_name, "foo")
self.assertEqual(service.directory_path, "bar/baz")
self.assertEqual(service.snapshot, "snap")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo/bar%2Fbaz?sharesnapshot=snap&'))
service = ShareDirectoryClient(account_url=custom_account_url, share_name='foo', directory_path="")
self.assertEqual(service.account_name, None)
self.assertEqual(service.share_name, "foo")
self.assertEqual(service.directory_path, "")
self.assertEqual(service.snapshot, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?'))
service = ShareFileClient(account_url=custom_account_url, share_name="foo", file_path="bar/baz/file", snapshot="snap")
self.assertEqual(service.account_name, None)
self.assertEqual(service.share_name, "foo")
self.assertEqual(service.directory_path, "bar/baz")
self.assertEqual(service.file_path, ["bar", "baz", "file"])
self.assertEqual(service.file_name, "file")
self.assertEqual(service.snapshot, "snap")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo/bar/baz/file?sharesnapshot=snap&'))
service = ShareFileClient(account_url=custom_account_url, share_name="foo", file_path="file")
self.assertEqual(service.account_name, None)
self.assertEqual(service.share_name, "foo")
self.assertEqual(service.directory_path, "")
self.assertEqual(service.file_path, ["file"])
self.assertEqual(service.file_name, "file")
self.assertEqual(service.snapshot, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo/file?'))
class FileStorageClient:
def __init__(self, credential: str, file_share_name: str, account_url: str):
self.directory_client = ShareDirectoryClient(
account_url=account_url,
credential=credential,
share_name=file_share_name,
directory_path=AZ_ML_ARTIFACT_DIRECTORY,
)
self.file_share = file_share_name
self.total_file_count = 1
self.uploaded_file_count = 0
try:
self.directory_client.create_directory()
except ResourceExistsError:
pass
self.subdirectory_client = None
def upload(self, source: str, asset_hash: str = None, show_progress: bool = True) -> str:
"""
Upload a file or directory to a path inside the file system
"""
asset_id = generate_asset_id(asset_hash, include_directory=False)
source_name = Path(source).name
dest = str(PurePosixPath(asset_id, source_name))
if not self.exists(asset_id):
# truncate path longer than 50 chars for terminal display
if show_progress and len(source_name) >= 50:
formatted_path = '{:.47}'.format(source_name) + "..."
else:
formatted_path = source_name
msg = f"Uploading {formatted_path}"
if os.path.isdir(source):
self.upload_dir(source, asset_id, msg=msg, show_progress=show_progress)
else:
self.upload_file(source, asset_id, msg=msg, show_progress=show_progress)
# upload must be completed before we try to generate confirmation file
while self.uploaded_file_count < self.total_file_count:
time.sleep(0.5)
self._set_confirmation_metadata(source, asset_id)
return dest
def upload_file(
self,
source: str,
dest: str,
show_progress: Optional[bool] = None,
msg: Optional[str] = None,
in_directory: bool = False,
subdirectory_client: Optional[ShareDirectoryClient] = None,
) -> None:
""" "
Upload a single file to a path inside the file system directory
"""
validate_content = os.stat(source).st_size > 0 # don't do checksum for empty files
with open(source, "rb") as data:
if in_directory:
file_name = dest.rsplit("/")[-1]
subdirectory_client.upload_file(file_name=file_name, data=data, validate_content=validate_content)
else:
iterable = tqdm(range(1), desc=msg) if show_progress else range(1)
for i in iterable:
self.directory_client.upload_file(file_name=dest, data=data,
validate_content=validate_content)
self.uploaded_file_count = self.uploaded_file_count + 1
def upload_dir(self, source: str, dest: str, msg: str, show_progress: bool) -> None:
"""
Upload a directory to a path inside the fileshare directory
"""
subdir = self.directory_client.create_subdirectory(dest)
prefix = "" if dest == "" else dest + "/"
prefix += os.path.basename(source) + "/"
self.total_file_count = sum(len(files) for _, _, files in os.walk(source))
if show_progress:
progress_bar = tqdm(total=self.total_file_count, desc=msg)
for root, dirs, files in os.walk(source):
if sys.platform.startswith(("win32", "cygwin")):
split_char = "\\"
else:
split_char = "/"
trunc_root = root.rsplit(split_char)[-1]
subdir = subdir.create_subdirectory(trunc_root)
upload_paths = traverse_directory(root, files, source, prefix)
for f, b in upload_paths:
self.upload_file(f, b, in_directory=True, subdirectory_client=subdir)
if show_progress:
progress_bar.update(1)
progress_bar.refresh()
if show_progress:
progress_bar.close()
def exists(self, asset_id: str) -> bool:
"""
Check if file or directory already exists in fileshare directory
"""
existing_items = {item["name"]: item["is_directory"] for item in
self.directory_client.list_directories_and_files()}
if asset_id in existing_items:
if existing_items[asset_id]:
client = self.directory_client.get_subdirectory_client(asset_id)
properties = client.get_directory_properties()
else:
client = self.directory_client.get_file_client(asset_id)
properties = client.get_file_properties()
if properties["metadata"] == UPLOAD_CONFIRMATION:
return True
else:
delete(client)
return False
def download(self, starts_with: str = "", destination: str = Path.home(), max_concurrency: int = 4) -> None:
"""
Downloads all contents inside a specified fileshare directory
"""
recursive_download(
client=self.directory_client,
starts_with=starts_with,
destination=destination,
max_concurrency=max_concurrency,
)
def _set_confirmation_metadata(self, source: str, dest: str):
if os.path.isdir(source):
properties = self.directory_client.get_subdirectory_client(dest)
properties.set_directory_metadata(UPLOAD_CONFIRMATION)
else:
properties = self.directory_client.get_file_client(dest)
properties.set_file_metadata(UPLOAD_CONFIRMATION)
with open(local_file_path, "wb") as file_handle:
data = file_client.download_file()
data.readinto(file_handle)
return 0
def ls_dir_files(this_share_name, this_dir_path):
"""
Input:
- func input: type(this_share_name) == type(str())
- func input: type(this_dir_path) == type(str())
- env:AZ_CONN_STR
Returns: list()
"""
parent_dir = ShareDirectoryClient.from_connection_string(
conn_str=AZ_CONN_STR,
share_name=this_share_name,
directory_path=this_dir_path
)
this_list = list(parent_dir.list_directories_and_files())
print(this_list)
return this_list
def set_logging_to_debug():
"""
Prints in stdout
Input:
- env:AZ_CONN_STR
Returns: Connection_object ShareServiceClient()
"""