def process():
action, mutex_url = get_raw_input()
# check whether the blob exists, if not quit right away to avoid wasting time
blob_client = BlobClient(mutex_url)
try:
blob_client.get_blob_properties()
print("INFO: validated mutex url")
except HttpResponseError as e:
raise ValueError('please provide an existing and valid blob URL, failed to get properties with error: ' + e)
# get a handle on the lease
lease_client = LeaseClient(blob_client)
if action == UNLOCK:
# make the lease free as soon as possible
lease_client.break_lease(lease_break_period=1)
print(f"INFO: successfully unlocked the mutex!")
return
# action is lock, attempt to acquire the lease continuously
while True:
# try to acquire and infinite lease
try:
lease_client.acquire(lease_duration=-1)
# if we get here, the acquire call succeeded
# if we don't get here it stalls forever, as expected
print(f"INFO: successfully locked the mutex!")
return
except HttpResponseError:
# failed to acquire lease, another agent holds the mutex
# sleep a bit (randomly) and try again
sleep_period = random.randint(1, 5)
print(f"INFO: failed to lock mutex, wait for {sleep_period} and try again")
time.sleep(sleep_period)
def __wait_for_copy(self, blob: BlobClient):
"""
Wait for the start_copy_from_url method to be completed
as per: https://github.com/Azure/azure-sdk-for-python/issues/7043
"""
count = 0
props = blob.get_blob_properties()
while props.copy.status == 'pending':
count = count + 1
if count > 10:
raise TimeoutError(
'Timed out waiting for async copy to complete.')
time.sleep(5)
props = blob.get_blob_properties()
return props
def set_blob_tier(blob: BlobClient, tier: StandardBlobTier, priority: RehydratePriority) -> None:
'''Set/change blob tier'''
log.info(f'Setting blob tier for {blob.blob_name} to {tier.value} with priority {priority.value}')
blob.set_standard_blob_tier(tier, rehydrate_priority=priority)
INFO = blob.get_blob_properties()
log.info(f'blob {blob.blob_name} tier is currently {INFO.blob_tier} and archive_status is now at '
f'{INFO.archive_status}')
def get(cls):
if cls.fse is None:
from azure.storage.blob import BlobClient
logger.debug("========PREPARING TO DOWNLOAD FSE MODEL")
storage_account_name = args.storage_account_name
container_client_credential = ClientSecretCredential(
tenant_id=args.directory_id,
client_id=args.application_id,
client_secret=SERVICE_PRINCIPAL_SECRET)
blob = BlobClient(
account_url=
f"https://{storage_account_name}.blob.core.windows.net",
container_name=args.domain_expert_container_name,
blob_name=args.fse_folder + "/" + args.fse_file,
credential=container_client_credential)
file_size = blob.get_blob_properties().size
if os.path.exists(args.fse_file) == False or (
os.path.exists(args.fse_file)
and os.stat(args.fse_file).st_size != file_size):
with open(args.fse_file, "wb") as my_blob:
blob_data = blob.download_blob()
blob_data.readinto(my_blob)
fse, ilist, jobs_lookup = dill.load(gzip.open(args.fse_file, "rb"))
cls.fse = fse
cls.ilist = ilist
cls.jobs_lookup = jobs_lookup
return cls.fse, cls.ilist, cls.jobs_lookup
def get_existing_metadata(blob_client: BlobClient):
try:
return exclude_key_prefix(blob_client.get_blob_properties().metadata,
FSS_METADATA_PREFIX)
except Exception as ex:
logging.warn(f'failed to get existing metadata: {ex}')
return {}
def get(cls, args, SERVICE_PRINCIPAL_SECRET, logger):
if "model" not in cls.gensim_model:
from azure.storage.blob import BlobClient
# logger.debug("========PREPARING TO SETUP GENSIM")
storage_account_name = args.storage_account_name
container_client_credential = ClientSecretCredential(
tenant_id=args.directory_id,
client_id=args.application_id,
client_secret=SERVICE_PRINCIPAL_SECRET)
blob = BlobClient(
account_url=
f"https://{storage_account_name}.blob.core.windows.net",
container_name=args.domain_expert_container_name,
blob_name=args.domain_expert_folder_path + "/" +
args.domain_expert_file_name,
credential=container_client_credential)
file_size = blob.get_blob_properties().size
if os.path.exists(args.domain_expert_file_name) is False or (
os.path.exists(args.domain_expert_file_name) and os.stat(
args.domain_expert_file_name).st_size != file_size):
print("getting domain expert")
with open(args.domain_expert_file_name, "wb") as my_blob:
if logger is not None:
logger.debug("downloading domain expert")
else:
print("downloading domain expert")
blob_data = blob.download_blob()
blob_data.readinto(my_blob)
# logger.debug("end downloading")
if logger is not None:
logger.debug("after domain expert download")
else:
print("after domain expert download")
from gensim.models import Word2Vec
try:
cls.gensim_model["model"] = Word2Vec.load(
args.domain_expert_file_name)
# logger.debug("========MODEL SET")
cls.gensim_model["vocab"] = cls.gensim_model["model"].wv.vocab
# logger.debug("========VOCAB SET")
except Exception as e:
if logger is not None:
logger.debug(
f"******************========== EXCEPTION ENCOUNTERED {e}"
)
else:
print(
f"******************========== EXCEPTION ENCOUNTERED {e}"
)
return cls.gensim_model
def test_upload(self):
blob = BlobClient(account_url=self.blob.storage_account_url,
container_name=self.blob.container,
blob_name=self.blob.blob_name,
credential=self.blob.credential)
try:
blob.get_blob_properties()
blob.delete_blob()
except ResourceNotFoundError:
pass
_logger.debug("TEST: Subir fichero sin borrar el local")
self.blob.upload(remove_local_data_file=False)
assert os.path.exists(
_data_path
), "Ha borrado el fichero local tras la subida a Blob Storage"
_logger.debug(
"TEST: Subir fichero que ya esta subido y se sobreescribe sin problemas y no se borra la copia local"
)
try:
self.blob.upload(remove_local_data_file=False)
except ResourceExistsError:
self.fail(
"Ha dado un error de blob existente, cuando deberia haberlo sobreescrito"
)
assert os.path.exists(
_data_path
), "Ha borrado el fichero local tras la subida a Blob storage"
_logger.debug(
"TEST: Se sube un fichero sobreescribiendo y borrando la copia local"
)
self.blob.upload()
assert not os.path.exists(_data_path)
def get(cls, args, SERVICE_PRINCIPAL_SECRET, logger):
if cls.big_taxo is None:
from azure.storage.blob import BlobClient
if logger is not None:
logger.debug("========PREPARING TO DOWNLOAD TAXNOMY")
else:
print("========PREPARING TO DOWNLOAD TAXNOMY")
storage_account_name = args.storage_account_name
container_client_credential = ClientSecretCredential(
tenant_id=args.directory_id,
client_id=args.application_id,
client_secret=SERVICE_PRINCIPAL_SECRET)
blob = BlobClient(
account_url=
f"https://{storage_account_name}.blob.core.windows.net",
container_name=args.domain_expert_container_name,
blob_name=args.domain_expert_folder_path + "/taxo/" +
"taxo_de.dill.gz",
credential=container_client_credential)
file_size = blob.get_blob_properties().size
if os.path.exists("taxo_de.dill.gz") is False or (
os.path.exists("taxo_de.dill.gz")
and os.stat("taxo_de.dill.gz").st_size != file_size):
with open("taxo_de.dill.gz", "wb") as my_blob:
# print("downloading")
blob_data = blob.download_blob()
blob_data.readinto(my_blob)
general_taxo_dict = dill.load(gzip.open("taxo_de.dill.gz", "rb"))
cls.big_taxo = general_taxo_dict
return cls.big_taxo
class PathClient(StorageAccountHostsMixin):
def __init__(
self, account_url, # type: str
file_system_name, # type: str
path_name, # type: str
credential=None, # type: Optional[Any]
**kwargs # type: Any
):
# type: (...) -> None
try:
if not account_url.lower().startswith('http'):
account_url = "https://" + account_url
except AttributeError:
raise ValueError("Account URL must be a string.")
parsed_url = urlparse(account_url.rstrip('/'))
# remove the preceding/trailing delimiter from the path components
file_system_name = file_system_name.strip('/')
path_name = path_name.strip('/')
if not (file_system_name and path_name):
raise ValueError("Please specify a container name and blob name.")
if not parsed_url.netloc:
raise ValueError("Invalid URL: {}".format(account_url))
blob_account_url = convert_dfs_url_to_blob_url(account_url)
datalake_hosts = kwargs.pop('_hosts', None)
blob_hosts = None
if datalake_hosts:
blob_primary_account_url = convert_dfs_url_to_blob_url(datalake_hosts[LocationMode.PRIMARY])
blob_secondary_account_url = convert_dfs_url_to_blob_url(datalake_hosts[LocationMode.SECONDARY])
blob_hosts = {LocationMode.PRIMARY: blob_primary_account_url,
LocationMode.SECONDARY: blob_secondary_account_url}
self._blob_client = BlobClient(blob_account_url, file_system_name, path_name,
credential=credential, _hosts=blob_hosts, **kwargs)
_, sas_token = parse_query(parsed_url.query)
self.file_system_name = file_system_name
self.path_name = path_name
self._query_str, self._raw_credential = self._format_query_string(sas_token, credential)
super(PathClient, self).__init__(parsed_url, service='dfs', credential=self._raw_credential,
_hosts=datalake_hosts, **kwargs)
self._client = DataLakeStorageClient(self.url, file_system_name, path_name, pipeline=self._pipeline)
def _format_url(self, hostname):
file_system_name = self.file_system_name
if isinstance(file_system_name, six.text_type):
file_system_name = file_system_name.encode('UTF-8')
return "{}://{}/{}/{}{}".format(
self.scheme,
hostname,
quote(file_system_name),
quote(self.path_name, safe='~'),
self._query_str)
def _create_path_options(self, resource_type, content_settings=None, metadata=None, **kwargs):
# type: (Optional[ContentSettings], Optional[Dict[str, str]], **Any) -> Dict[str, Any]
if self.require_encryption or (self.key_encryption_key is not None):
raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
access_conditions = get_access_conditions(kwargs.pop('lease', None))
mod_conditions = get_mod_conditions(kwargs)
path_http_headers = None
if content_settings:
path_http_headers = get_path_http_headers(content_settings)
options = {
'resource': resource_type,
'properties': add_metadata_headers(metadata),
'permissions': kwargs.pop('permissions', None),
'umask': kwargs.pop('umask', None),
'path_http_headers': path_http_headers,
'lease_access_conditions': access_conditions,
'modified_access_conditions': mod_conditions,
'timeout': kwargs.pop('timeout', None),
'cls': return_response_headers}
options.update(kwargs)
return options
def _create(self, resource_type, content_settings=None, metadata=None, **kwargs):
# type: (...) -> Dict[str, Union[str, datetime]]
"""
Create directory or file
:param resource_type: Required for Create File and Create Directory.
The value must be "file" or "directory". Possible values include:
'directory', 'file'
:type resource_type: str
:param ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:param metadata:
Name-value pairs associated with the file/directory as metadata.
:type metadata: dict(str, str)
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword str umask: Optional and only valid if Hierarchical Namespace is enabled for the account.
When creating a file or directory and the parent folder does not have a default ACL,
the umask restricts the permissions of the file or directory to be created.
The resulting permission is given by p & ^u, where p is the permission and u is the umask.
For example, if p is 0777 and u is 0057, then the resulting permission is 0720.
The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027.
The umask must be specified in 4-digit octal notation (e.g. 0766).
:keyword permissions: Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
:type permissions: str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:keyword str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: Dict[str, Union[str, datetime]]
"""
options = self._create_path_options(
resource_type,
content_settings=content_settings,
metadata=metadata,
**kwargs)
try:
return self._client.path.create(**options)
except StorageErrorException as error:
raise error
@staticmethod
def _delete_path_options(**kwargs):
# type: (Optional[ContentSettings], Optional[Dict[str, str]], **Any) -> Dict[str, Any]
access_conditions = get_access_conditions(kwargs.pop('lease', None))
mod_conditions = get_mod_conditions(kwargs)
options = {
'recursive': True,
'lease_access_conditions': access_conditions,
'modified_access_conditions': mod_conditions,
'timeout': kwargs.pop('timeout', None)}
options.update(kwargs)
return options
def _delete(self, **kwargs):
# type: (bool, **Any) -> None
"""
Marks the specified path for deletion.
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:param ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:param ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:param str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:param str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:param int timeout:
The timeout parameter is expressed in seconds.
:return: None
"""
options = self._delete_path_options(**kwargs)
try:
return self._client.path.delete(**options)
except StorageErrorException as error:
raise error
@staticmethod
def _set_access_control_options(owner=None, group=None, permissions=None, acl=None, **kwargs):
# type: (Optional[ContentSettings], Optional[Dict[str, str]], **Any) -> Dict[str, Any]
access_conditions = get_access_conditions(kwargs.pop('lease', None))
mod_conditions = get_mod_conditions(kwargs)
options = {
'owner': owner,
'group': group,
'permissions': permissions,
'acl': acl,
'lease_access_conditions': access_conditions,
'modified_access_conditions': mod_conditions,
'timeout': kwargs.pop('timeout', None),
'cls': return_response_headers}
options.update(kwargs)
return options
def set_access_control(self, owner=None, # type: Optional[str]
group=None, # type: Optional[str]
permissions=None, # type: Optional[str]
acl=None, # type: Optional[str]
**kwargs):
# type: (...) -> Dict[str, Union[str, datetime]]
"""
Set the owner, group, permissions, or access control list for a path.
:param owner: Optional. The owner of the file or directory.
:type owner: str
:param group: Optional. The owning group of the file or directory.
:type group: str
:param permissions: Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
:type permissions: str
:param acl: Sets POSIX access control rights on files and directories.
The value is a comma-separated list of access control entries. Each
access control entry (ACE) consists of a scope, a type, a user or
group identifier, and permissions in the format
"[scope:][type]:[id]:[permissions]".
:type acl: str
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:keyword str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:keyword: response dict (Etag and last modified).
"""
options = self._set_access_control_options(owner=owner, group=group, permissions=permissions, acl=acl, **kwargs)
try:
return self._client.path.set_access_control(**options)
except StorageErrorException as error:
raise error
@staticmethod
def _get_access_control_options(upn=None, # type: Optional[bool]
**kwargs):
# type: (...) -> Dict[str, Any]
access_conditions = get_access_conditions(kwargs.pop('lease', None))
mod_conditions = get_mod_conditions(kwargs)
options = {
'action': 'getAccessControl',
'upn': upn if upn else False,
'lease_access_conditions': access_conditions,
'modified_access_conditions': mod_conditions,
'timeout': kwargs.pop('timeout', None),
'cls': return_response_headers}
options.update(kwargs)
return options
def get_access_control(self, upn=None, # type: Optional[bool]
**kwargs):
# type: (...) -> Dict[str, Any]
"""
:param upn: Optional. Valid only when Hierarchical Namespace is
enabled for the account. If "true", the user identity values returned
in the x-ms-owner, x-ms-group, and x-ms-acl response headers will be
transformed from Azure Active Directory Object IDs to User Principal
Names. If "false", the values will be returned as Azure Active
Directory Object IDs. The default value is false. Note that group and
application Object IDs are not translated because they do not have
unique friendly names.
:type upn: bool
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:keyword str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:keyword: response dict.
"""
options = self._get_access_control_options(upn=upn, **kwargs)
try:
return self._client.path.get_properties(**options)
except StorageErrorException as error:
raise error
def _rename_path_options(self, rename_source, content_settings=None, metadata=None, **kwargs):
# type: (Optional[ContentSettings], Optional[Dict[str, str]], **Any) -> Dict[str, Any]
if self.require_encryption or (self.key_encryption_key is not None):
raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
access_conditions = get_access_conditions(kwargs.pop('lease', None))
source_lease_id = get_lease_id(kwargs.pop('source_lease', None))
mod_conditions = get_mod_conditions(kwargs)
source_mod_conditions = get_source_mod_conditions(kwargs)
path_http_headers = None
if content_settings:
path_http_headers = get_path_http_headers(content_settings)
options = {
'rename_source': rename_source,
'properties': add_metadata_headers(metadata),
'permissions': kwargs.pop('permissions', None),
'umask': kwargs.pop('umask', None),
'path_http_headers': path_http_headers,
'lease_access_conditions': access_conditions,
'source_lease_id': source_lease_id,
'modified_access_conditions': mod_conditions,
'source_modified_access_conditions':source_mod_conditions,
'timeout': kwargs.pop('timeout', None),
'mode': 'legacy',
'cls': return_response_headers}
options.update(kwargs)
return options
def _rename_path(self, rename_source,
**kwargs):
# type: (**Any) -> Dict[str, Any]
"""
Rename directory or file
:param rename_source: The value must have the following format: "/{filesystem}/{path}".
:type rename_source: str
:param source_lease: A lease ID for the source path. If specified,
the source path must have an active lease and the leaase ID must
match.
:type source_lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:param ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:param lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:param str umask: Optional and only valid if Hierarchical Namespace is enabled for the account.
When creating a file or directory and the parent folder does not have a default ACL,
the umask restricts the permissions of the file or directory to be created.
The resulting permission is given by p & ^u, where p is the permission and u is the umask.
For example, if p is 0777 and u is 0057, then the resulting permission is 0720.
The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027.
The umask must be specified in 4-digit octal notation (e.g. 0766).
:param permissions: Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
:type permissions: str
:param ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:param ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:param str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:param str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:param ~datetime.datetime source_if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:param ~datetime.datetime source_if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:param str source_if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:param str source_if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:param int timeout:
The timeout parameter is expressed in seconds.
:return:
"""
options = self._rename_path_options(
rename_source,
**kwargs)
try:
return self._client.path.create(**options)
except StorageErrorException as error:
raise error
def _get_path_properties(self, **kwargs):
# type: (**Any) -> Union[FileProperties, DirectoryProperties]
"""Returns all user-defined metadata, standard HTTP properties, and
system properties for the file or directory. It does not return the content of the directory or file.
:keyword lease:
Required if the directory or file has an active lease. Value can be a DataLakeLeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword :class:`MatchConditions` match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:rtype: DirectoryProperties or FileProperties
.. admonition:: Example:
.. literalinclude:: ../tests/test_blob_samples_common.py
:start-after: [START get_blob_properties]
:end-before: [END get_blob_properties]
:language: python
:dedent: 8
:caption: Getting the properties for a file/directory.
"""
path_properties = self._blob_client.get_blob_properties(**kwargs)
path_properties.__class__ = DirectoryProperties
return path_properties
def set_metadata(self, metadata=None, # type: Optional[Dict[str, str]]
**kwargs):
# type: (...) -> Dict[str, Union[str, datetime]]
"""Sets one or more user-defined name-value pairs for the specified
file system. Each call to this operation replaces all existing metadata
attached to the file system. To remove all metadata from the file system,
call this operation with no metadata dict.
:param metadata:
A dict containing name-value pairs to associate with the file system as
metadata. Example: {'category':'test'}
:type metadata: dict[str, str]
:keyword str or ~azure.storage.filedatalake.DataLakeLeaseClient lease:
If specified, set_file_system_metadata only succeeds if the
file system's lease is active and matches this ID.
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword :class:`MatchConditions` match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:returns: file system-updated property dict (Etag and last modified).
.. admonition:: Example:
.. literalinclude:: ../samples/test_file_system_samples.py
:start-after: [START set_file_system_metadata]
:end-before: [END set_file_system_metadata]
:language: python
:dedent: 12
:caption: Setting metadata on the container.
"""
return self._blob_client.set_blob_metadata(metadata=metadata, **kwargs)
def set_http_headers(self, content_settings=None, # type: Optional[ContentSettings]
**kwargs):
# type: (...) -> Dict[str, Any]
"""Sets system properties on the file or directory.
If one property is set for the content_settings, all properties will be overriden.
:param ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set file/directory properties.
:keyword str or ~azure.storage.filedatalake.DataLakeLeaseClient lease:
If specified, set_file_system_metadata only succeeds if the
file system's lease is active and matches this ID.
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword :class:`MatchConditions` match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:returns: file/directory-updated property dict (Etag and last modified)
:rtype: Dict[str, Any]
"""
return self._blob_client.set_http_headers(content_settings=content_settings, **kwargs)
def acquire_lease(self, lease_duration=-1, # type: Optional[int]
lease_id=None, # type: Optional[str]
**kwargs):
# type: (...) -> DataLakeLeaseClient
"""
Requests a new lease. If the file or directory does not have an active lease,
the DataLake service creates a lease on the file/directory and returns a new
lease ID.
:param int lease_duration:
Specifies the duration of the lease, in seconds, or negative one
(-1) for a lease that never expires. A non-infinite lease can be
between 15 and 60 seconds. A lease duration cannot be changed
using renew or change. Default is -1 (infinite lease).
:param str lease_id:
Proposed lease ID, in a GUID string format. The DataLake service returns
400 (Invalid request) if the proposed lease ID is not in the correct format.
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword :class:`MatchConditions` match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:returns: A DataLakeLeaseClient object, that can be run in a context manager.
:rtype: ~azure.storage.filedatalake.DataLakeLeaseClient
.. admonition:: Example:
.. literalinclude:: ../samples/test_file_system_samples.py
:start-after: [START acquire_lease_on_file_system]
:end-before: [END acquire_lease_on_file_system]
:language: python
:dedent: 8
:caption: Acquiring a lease on the file_system.
"""
lease = DataLakeLeaseClient(self, lease_id=lease_id) # type: ignore
lease.acquire(lease_duration=lease_duration, **kwargs)
return lease
class PathClient(StorageAccountHostsMixin):
def __init__(
self,
account_url, # type: str
file_system_name, # type: str
path_name, # type: str
credential=None, # type: Optional[Any]
**kwargs # type: Any
):
# type: (...) -> None
try:
if not account_url.lower().startswith('http'):
account_url = "https://" + account_url
except AttributeError:
raise ValueError("Account URL must be a string.")
parsed_url = urlparse(account_url.rstrip('/'))
# remove the preceding/trailing delimiter from the path components
file_system_name = file_system_name.strip('/')
# the name of root directory is /
if path_name != '/':
path_name = path_name.strip('/')
if not (file_system_name and path_name):
raise ValueError(
"Please specify a file system name and file path.")
if not parsed_url.netloc:
raise ValueError("Invalid URL: {}".format(account_url))
blob_account_url = convert_dfs_url_to_blob_url(account_url)
self._blob_account_url = blob_account_url
datalake_hosts = kwargs.pop('_hosts', None)
blob_hosts = None
if datalake_hosts:
blob_primary_account_url = convert_dfs_url_to_blob_url(
datalake_hosts[LocationMode.PRIMARY])
blob_hosts = {
LocationMode.PRIMARY: blob_primary_account_url,
LocationMode.SECONDARY: ""
}
self._blob_client = BlobClient(blob_account_url,
file_system_name,
path_name,
credential=credential,
_hosts=blob_hosts,
**kwargs)
_, sas_token = parse_query(parsed_url.query)
self.file_system_name = file_system_name
self.path_name = path_name
self._query_str, self._raw_credential = self._format_query_string(
sas_token, credential)
super(PathClient, self).__init__(parsed_url,
service='dfs',
credential=self._raw_credential,
_hosts=datalake_hosts,
**kwargs)
# ADLS doesn't support secondary endpoint, make sure it's empty
self._hosts[LocationMode.SECONDARY] = ""
api_version = get_api_version(kwargs)
self._client = AzureDataLakeStorageRESTAPI(
self.url,
base_url=self.url,
file_system=file_system_name,
path=path_name,
pipeline=self._pipeline)
self._client._config.version = api_version # pylint: disable=protected-access
self._datalake_client_for_blob_operation = AzureDataLakeStorageRESTAPI(
self._blob_client.url,
base_url=self._blob_client.url,
file_system=file_system_name,
path=path_name,
pipeline=self._pipeline)
self._datalake_client_for_blob_operation._config.version = api_version # pylint: disable=protected-access
def __exit__(self, *args):
self._blob_client.close()
super(PathClient, self).__exit__(*args)
def close(self):
# type: () -> None
""" This method is to close the sockets opened by the client.
It need not be used when using with a context manager.
"""
self._blob_client.close()
self.__exit__()
def _format_url(self, hostname):
file_system_name = self.file_system_name
if isinstance(file_system_name, six.text_type):
file_system_name = file_system_name.encode('UTF-8')
return "{}://{}/{}/{}{}".format(self.scheme, hostname,
quote(file_system_name),
quote(self.path_name, safe='~'),
self._query_str)
def _create_path_options(
self,
resource_type,
content_settings=None, # type: Optional[ContentSettings]
metadata=None, # type: Optional[Dict[str, str]]
**kwargs):
# type: (...) -> Dict[str, Any]
if self.require_encryption or (self.key_encryption_key is not None):
raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
access_conditions = get_access_conditions(kwargs.pop('lease', None))
mod_conditions = get_mod_conditions(kwargs)
path_http_headers = None
if content_settings:
path_http_headers = get_path_http_headers(content_settings)
options = {
'resource': resource_type,
'properties': add_metadata_headers(metadata),
'permissions': kwargs.pop('permissions', None),
'umask': kwargs.pop('umask', None),
'path_http_headers': path_http_headers,
'lease_access_conditions': access_conditions,
'modified_access_conditions': mod_conditions,
'timeout': kwargs.pop('timeout', None),
'cls': return_response_headers
}
options.update(kwargs)
return options
def _create(self,
resource_type,
content_settings=None,
metadata=None,
**kwargs):
# type: (...) -> Dict[str, Union[str, datetime]]
"""
Create directory or file
:param resource_type:
Required for Create File and Create Directory.
The value must be "file" or "directory". Possible values include:
'directory', 'file'
:type resource_type: str
:param ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:param metadata:
Name-value pairs associated with the file/directory as metadata.
:type metadata: dict(str, str)
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword str umask:
Optional and only valid if Hierarchical Namespace is enabled for the account.
When creating a file or directory and the parent folder does not have a default ACL,
the umask restricts the permissions of the file or directory to be created.
The resulting permission is given by p & ^u, where p is the permission and u is the umask.
For example, if p is 0777 and u is 0057, then the resulting permission is 0720.
The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027.
The umask must be specified in 4-digit octal notation (e.g. 0766).
:keyword permissions:
Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
:type permissions: str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: Dict[str, Union[str, datetime]]
"""
options = self._create_path_options(resource_type,
content_settings=content_settings,
metadata=metadata,
**kwargs)
try:
return self._client.path.create(**options)
except HttpResponseError as error:
process_storage_error(error)
@staticmethod
def _delete_path_options(**kwargs):
# type: (**Any) -> Dict[str, Any]
access_conditions = get_access_conditions(kwargs.pop('lease', None))
mod_conditions = get_mod_conditions(kwargs)
options = {
'lease_access_conditions': access_conditions,
'modified_access_conditions': mod_conditions,
'cls': return_response_headers,
'timeout': kwargs.pop('timeout', None)
}
options.update(kwargs)
return options
def _delete(self, **kwargs):
# type: (**Any) -> Dict[Union[datetime, str]]
"""
Marks the specified path for deletion.
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:param ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:param ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:param int timeout:
The timeout parameter is expressed in seconds.
:return: None
"""
options = self._delete_path_options(**kwargs)
try:
return self._client.path.delete(**options)
except HttpResponseError as error:
process_storage_error(error)
@staticmethod
def _set_access_control_options(owner=None,
group=None,
permissions=None,
acl=None,
**kwargs):
# type: (...) -> Dict[str, Any]
access_conditions = get_access_conditions(kwargs.pop('lease', None))
mod_conditions = get_mod_conditions(kwargs)
options = {
'owner': owner,
'group': group,
'permissions': permissions,
'acl': acl,
'lease_access_conditions': access_conditions,
'modified_access_conditions': mod_conditions,
'timeout': kwargs.pop('timeout', None),
'cls': return_response_headers
}
options.update(kwargs)
return options
def set_access_control(
self,
owner=None, # type: Optional[str]
group=None, # type: Optional[str]
permissions=None, # type: Optional[str]
acl=None, # type: Optional[str]
**kwargs):
# type: (...) -> Dict[str, Union[str, datetime]]
"""
Set the owner, group, permissions, or access control list for a path.
:param owner:
Optional. The owner of the file or directory.
:type owner: str
:param group:
Optional. The owning group of the file or directory.
:type group: str
:param permissions:
Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
permissions and acl are mutually exclusive.
:type permissions: str
:param acl:
Sets POSIX access control rights on files and directories.
The value is a comma-separated list of access control entries. Each
access control entry (ACE) consists of a scope, a type, a user or
group identifier, and permissions in the format
"[scope:][type]:[id]:[permissions]".
permissions and acl are mutually exclusive.
:type acl: str
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:keyword: response dict (Etag and last modified).
"""
if not any([owner, group, permissions, acl]):
raise ValueError(
"At least one parameter should be set for set_access_control API"
)
options = self._set_access_control_options(owner=owner,
group=group,
permissions=permissions,
acl=acl,
**kwargs)
try:
return self._client.path.set_access_control(**options)
except HttpResponseError as error:
process_storage_error(error)
@staticmethod
def _get_access_control_options(
upn=None, # type: Optional[bool]
**kwargs):
# type: (...) -> Dict[str, Any]
access_conditions = get_access_conditions(kwargs.pop('lease', None))
mod_conditions = get_mod_conditions(kwargs)
options = {
'action': 'getAccessControl',
'upn': upn if upn else False,
'lease_access_conditions': access_conditions,
'modified_access_conditions': mod_conditions,
'timeout': kwargs.pop('timeout', None),
'cls': return_response_headers
}
options.update(kwargs)
return options
def get_access_control(
self,
upn=None, # type: Optional[bool]
**kwargs):
# type: (...) -> Dict[str, Any]
"""
:param upn: Optional.
Valid only when Hierarchical Namespace is
enabled for the account. If "true", the user identity values returned
in the x-ms-owner, x-ms-group, and x-ms-acl response headers will be
transformed from Azure Active Directory Object IDs to User Principal
Names. If "false", the values will be returned as Azure Active
Directory Object IDs. The default value is false. Note that group and
application Object IDs are not translated because they do not have
unique friendly names.
:type upn: bool
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:keyword: response dict.
"""
options = self._get_access_control_options(upn=upn, **kwargs)
try:
return self._client.path.get_properties(**options)
except HttpResponseError as error:
process_storage_error(error)
@staticmethod
def _set_access_control_recursive_options(mode, acl, **kwargs):
# type: (str, str, **Any) -> Dict[str, Any]
options = {
'mode': mode,
'force_flag': kwargs.pop('continue_on_failure', None),
'timeout': kwargs.pop('timeout', None),
'continuation': kwargs.pop('continuation_token', None),
'max_records': kwargs.pop('batch_size', None),
'acl': acl,
'cls': return_headers_and_deserialized
}
options.update(kwargs)
return options
def set_access_control_recursive(self, acl, **kwargs):
# type: (str, **Any) -> AccessControlChangeResult
"""
Sets the Access Control on a path and sub-paths.
:param acl:
Sets POSIX access control rights on files and directories.
The value is a comma-separated list of access control entries. Each
access control entry (ACE) consists of a scope, a type, a user or
group identifier, and permissions in the format
"[scope:][type]:[id]:[permissions]".
:type acl: str
:keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
Callback where the caller can track progress of the operation
as well as collect paths that failed to change Access Control.
:keyword str continuation_token:
Optional continuation token that can be used to resume previously stopped operation.
:keyword int batch_size:
Optional. If data set size exceeds batch size then operation will be split into multiple
requests so that progress can be tracked. Batch size should be between 1 and 2000.
The default when unspecified is 2000.
:keyword int max_batches:
Optional. Defines maximum number of batches that single change Access Control operation can execute.
If maximum is reached before all sub-paths are processed,
then continuation token can be used to resume operation.
Empty value indicates that maximum number of batches in unbound and operation continues till end.
:keyword bool continue_on_failure:
If set to False, the operation will terminate quickly on encountering user errors (4XX).
If True, the operation will ignore user errors and proceed with the operation on other sub-entities of
the directory.
Continuation token will only be returned when continue_on_failure is True in case of user errors.
If not set the default value is False for this.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: A summary of the recursive operations, including the count of successes and failures,
as well as a continuation token in case the operation was terminated prematurely.
:rtype: :class:`~azure.storage.filedatalake.AccessControlChangeResult`
:raises ~azure.core.exceptions.AzureError:
User can restart the operation using continuation_token field of AzureError if the token is available.
"""
if not acl:
raise ValueError(
"The Access Control List must be set for this operation")
progress_hook = kwargs.pop('progress_hook', None)
max_batches = kwargs.pop('max_batches', None)
options = self._set_access_control_recursive_options(mode='set',
acl=acl,
**kwargs)
return self._set_access_control_internal(options=options,
progress_hook=progress_hook,
max_batches=max_batches)
def update_access_control_recursive(self, acl, **kwargs):
# type: (str, **Any) -> AccessControlChangeResult
"""
Modifies the Access Control on a path and sub-paths.
:param acl:
Modifies POSIX access control rights on files and directories.
The value is a comma-separated list of access control entries. Each
access control entry (ACE) consists of a scope, a type, a user or
group identifier, and permissions in the format
"[scope:][type]:[id]:[permissions]".
:type acl: str
:keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
Callback where the caller can track progress of the operation
as well as collect paths that failed to change Access Control.
:keyword str continuation_token:
Optional continuation token that can be used to resume previously stopped operation.
:keyword int batch_size:
Optional. If data set size exceeds batch size then operation will be split into multiple
requests so that progress can be tracked. Batch size should be between 1 and 2000.
The default when unspecified is 2000.
:keyword int max_batches:
Optional. Defines maximum number of batches that single change Access Control operation can execute.
If maximum is reached before all sub-paths are processed,
then continuation token can be used to resume operation.
Empty value indicates that maximum number of batches in unbound and operation continues till end.
:keyword bool continue_on_failure:
If set to False, the operation will terminate quickly on encountering user errors (4XX).
If True, the operation will ignore user errors and proceed with the operation on other sub-entities of
the directory.
Continuation token will only be returned when continue_on_failure is True in case of user errors.
If not set the default value is False for this.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: A summary of the recursive operations, including the count of successes and failures,
as well as a continuation token in case the operation was terminated prematurely.
:rtype: :class:`~azure.storage.filedatalake.AccessControlChangeResult`
:raises ~azure.core.exceptions.AzureError:
User can restart the operation using continuation_token field of AzureError if the token is available.
"""
if not acl:
raise ValueError(
"The Access Control List must be set for this operation")
progress_hook = kwargs.pop('progress_hook', None)
max_batches = kwargs.pop('max_batches', None)
options = self._set_access_control_recursive_options(mode='modify',
acl=acl,
**kwargs)
return self._set_access_control_internal(options=options,
progress_hook=progress_hook,
max_batches=max_batches)
def remove_access_control_recursive(self, acl, **kwargs):
# type: (str, **Any) -> AccessControlChangeResult
"""
Removes the Access Control on a path and sub-paths.
:param acl:
Removes POSIX access control rights on files and directories.
The value is a comma-separated list of access control entries. Each
access control entry (ACE) consists of a scope, a type, and a user or
group identifier in the format "[scope:][type]:[id]".
:type acl: str
:keyword func(~azure.storage.filedatalake.AccessControlChanges) progress_hook:
Callback where the caller can track progress of the operation
as well as collect paths that failed to change Access Control.
:keyword str continuation_token:
Optional continuation token that can be used to resume previously stopped operation.
:keyword int batch_size:
Optional. If data set size exceeds batch size then operation will be split into multiple
requests so that progress can be tracked. Batch size should be between 1 and 2000.
The default when unspecified is 2000.
:keyword int max_batches:
Optional. Defines maximum number of batches that single change Access Control operation can execute.
If maximum is reached before all sub-paths are processed then,
continuation token can be used to resume operation.
Empty value indicates that maximum number of batches in unbound and operation continues till end.
:keyword bool continue_on_failure:
If set to False, the operation will terminate quickly on encountering user errors (4XX).
If True, the operation will ignore user errors and proceed with the operation on other sub-entities of
the directory.
Continuation token will only be returned when continue_on_failure is True in case of user errors.
If not set the default value is False for this.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:return: A summary of the recursive operations, including the count of successes and failures,
as well as a continuation token in case the operation was terminated prematurely.
:rtype: :class:`~azure.storage.filedatalake.AccessControlChangeResult`
:raises ~azure.core.exceptions.AzureError:
User can restart the operation using continuation_token field of AzureError if the token is available.
"""
if not acl:
raise ValueError(
"The Access Control List must be set for this operation")
progress_hook = kwargs.pop('progress_hook', None)
max_batches = kwargs.pop('max_batches', None)
options = self._set_access_control_recursive_options(mode='remove',
acl=acl,
**kwargs)
return self._set_access_control_internal(options=options,
progress_hook=progress_hook,
max_batches=max_batches)
def _set_access_control_internal(self,
options,
progress_hook,
max_batches=None):
try:
continue_on_failure = options.get('force_flag')
total_directories_successful = 0
total_files_success = 0
total_failure_count = 0
batch_count = 0
last_continuation_token = None
current_continuation_token = None
continue_operation = True
while continue_operation:
headers, resp = self._client.path.set_access_control_recursive(
**options)
# make a running tally so that we can report the final results
total_directories_successful += resp.directories_successful
total_files_success += resp.files_successful
total_failure_count += resp.failure_count
batch_count += 1
current_continuation_token = headers['continuation']
if current_continuation_token is not None:
last_continuation_token = current_continuation_token
if progress_hook is not None:
progress_hook(
AccessControlChanges(
batch_counters=AccessControlChangeCounters(
directories_successful=resp.
directories_successful,
files_successful=resp.files_successful,
failure_count=resp.failure_count,
),
aggregate_counters=AccessControlChangeCounters(
directories_successful=
total_directories_successful,
files_successful=total_files_success,
failure_count=total_failure_count,
),
batch_failures=[
AccessControlChangeFailure(
name=failure.name,
is_directory=failure.type == 'DIRECTORY',
error_message=failure.error_message)
for failure in resp.failed_entries
],
continuation=last_continuation_token))
# update the continuation token, if there are more operations that cannot be completed in a single call
max_batches_satisfied = (max_batches is not None
and batch_count == max_batches)
continue_operation = bool(
current_continuation_token) and not max_batches_satisfied
options['continuation'] = current_continuation_token
# currently the service stops on any failure, so we should send back the last continuation token
# for the user to retry the failed updates
# otherwise we should just return what the service gave us
return AccessControlChangeResult(
counters=AccessControlChangeCounters(
directories_successful=total_directories_successful,
files_successful=total_files_success,
failure_count=total_failure_count),
continuation=last_continuation_token if total_failure_count > 0
and not continue_on_failure else current_continuation_token)
except HttpResponseError as error:
error.continuation_token = last_continuation_token
process_storage_error(error)
except AzureError as error:
error.continuation_token = last_continuation_token
raise error
def _rename_path_options(
self,
rename_source,
content_settings=None, # type: Optional[ContentSettings]
metadata=None, # type: Optional[Dict[str, str]]
**kwargs):
# type: (...) -> Dict[str, Any]
if self.require_encryption or (self.key_encryption_key is not None):
raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
if metadata or kwargs.pop('permissions', None) or kwargs.pop(
'umask', None):
raise ValueError(
"metadata, permissions, umask is not supported for this operation"
)
access_conditions = get_access_conditions(kwargs.pop('lease', None))
source_lease_id = get_lease_id(kwargs.pop('source_lease', None))
mod_conditions = get_mod_conditions(kwargs)
source_mod_conditions = get_source_mod_conditions(kwargs)
path_http_headers = None
if content_settings:
path_http_headers = get_path_http_headers(content_settings)
options = {
'rename_source': rename_source,
'path_http_headers': path_http_headers,
'lease_access_conditions': access_conditions,
'source_lease_id': source_lease_id,
'modified_access_conditions': mod_conditions,
'source_modified_access_conditions': source_mod_conditions,
'timeout': kwargs.pop('timeout', None),
'mode': 'legacy',
'cls': return_response_headers
}
options.update(kwargs)
return options
def _rename_path(self, rename_source, **kwargs):
# type: (str, **Any) -> Dict[str, Any]
"""
Rename directory or file
:param rename_source:
The value must have the following format: "/{filesystem}/{path}".
:type rename_source: str
:keyword ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:keyword source_lease:
A lease ID for the source path. If specified,
the source path must have an active lease and the leaase ID must
match.
:paramtype source_lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword lease:
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:keyword ~datetime.datetime source_if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime source_if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str source_etag:
The source ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions source_match_condition:
The source match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
"""
options = self._rename_path_options(rename_source, **kwargs)
try:
return self._client.path.create(**options)
except HttpResponseError as error:
process_storage_error(error)
def _get_path_properties(self, **kwargs):
# type: (**Any) -> Union[FileProperties, DirectoryProperties]
"""Returns all user-defined metadata, standard HTTP properties, and
system properties for the file or directory. It does not return the content of the directory or file.
:keyword lease:
Required if the directory or file has an active lease. Value can be a DataLakeLeaseClient object
or the lease ID as a string.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:rtype: DirectoryProperties or FileProperties
.. admonition:: Example:
.. literalinclude:: ../tests/test_blob_samples_common.py
:start-after: [START get_blob_properties]
:end-before: [END get_blob_properties]
:language: python
:dedent: 8
:caption: Getting the properties for a file/directory.
"""
path_properties = self._blob_client.get_blob_properties(**kwargs)
return path_properties
def _exists(self, **kwargs):
# type: (**Any) -> bool
"""
Returns True if a path exists and returns False otherwise.
:kwarg int timeout:
The timeout parameter is expressed in seconds.
:returns: boolean
"""
return self._blob_client.exists(**kwargs)
def set_metadata(
self,
metadata, # type: Dict[str, str]
**kwargs):
# type: (...) -> Dict[str, Union[str, datetime]]
"""Sets one or more user-defined name-value pairs for the specified
file system. Each call to this operation replaces all existing metadata
attached to the file system. To remove all metadata from the file system,
call this operation with no metadata dict.
:param metadata:
A dict containing name-value pairs to associate with the file system as
metadata. Example: {'category':'test'}
:type metadata: dict[str, str]
:keyword lease:
If specified, set_file_system_metadata only succeeds if the
file system's lease is active and matches this ID.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:returns: file system-updated property dict (Etag and last modified).
"""
return self._blob_client.set_blob_metadata(metadata=metadata, **kwargs)
def set_http_headers(
self,
content_settings=None, # type: Optional[ContentSettings]
**kwargs):
# type: (...) -> Dict[str, Any]
"""Sets system properties on the file or directory.
If one property is set for the content_settings, all properties will be overriden.
:param ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set file/directory properties.
:keyword lease:
If specified, set_file_system_metadata only succeeds if the
file system's lease is active and matches this ID.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:returns: file/directory-updated property dict (Etag and last modified)
:rtype: Dict[str, Any]
"""
return self._blob_client.set_http_headers(
content_settings=content_settings, **kwargs)
def acquire_lease(
self,
lease_duration=-1, # type: Optional[int]
lease_id=None, # type: Optional[str]
**kwargs):
# type: (...) -> DataLakeLeaseClient
"""
Requests a new lease. If the file or directory does not have an active lease,
the DataLake service creates a lease on the file/directory and returns a new
lease ID.
:param int lease_duration:
Specifies the duration of the lease, in seconds, or negative one
(-1) for a lease that never expires. A non-infinite lease can be
between 15 and 60 seconds. A lease duration cannot be changed
using renew or change. Default is -1 (infinite lease).
:param str lease_id:
Proposed lease ID, in a GUID string format. The DataLake service returns
400 (Invalid request) if the proposed lease ID is not in the correct format.
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:keyword ~datetime.datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:keyword str etag:
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
and act according to the condition specified by the `match_condition` parameter.
:keyword ~azure.core.MatchConditions match_condition:
The match condition to use upon the etag.
:keyword int timeout:
The timeout parameter is expressed in seconds.
:returns: A DataLakeLeaseClient object, that can be run in a context manager.
:rtype: ~azure.storage.filedatalake.DataLakeLeaseClient
"""
lease = DataLakeLeaseClient(self, lease_id=lease_id) # type: ignore
lease.acquire(lease_duration=lease_duration, **kwargs)
return lease
def copy_files(self, from_location, to_location, file_pattern):
result = messages.message["copy_files_failed"]
result, sources, source_names = self.list_files(location=from_location, file_pattern=file_pattern)
if sources is None:
print("no files found in source >" + from_location + "<.")
result = messages.message["ok"]
result["reference"] = "No files in source: " + from_location
return result
for file in sources:
# https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-copy?tabs=python
src_file = file # os.path.basename(file)
tgt_file = to_location + "/" + os.path.basename(src_file)
src_blob = BlobClient(
self.blob_service_client.url,
container_name=self.settings.storage_container,
blob_name=src_file,
credential=self.sas_token
)
lease = BlobLeaseClient(src_blob)
lease.acquire()
source_props = src_blob.get_blob_properties()
print("Lease state for source file %s: %s" %(src_file ,source_props.lease.state))
print(f"Copying %s.%s to %s using url %s"
% (self.settings.storage_container, src_file, tgt_file, src_blob.url))
self.tgt = self.blob_service_client.get_blob_client(self.settings.storage_container, tgt_file)
# download_file_path = os.path.join(".", str.replace("tryout", '.txt', 'DOWNLOAD.txt'))
# print("\nDownloading blob to \n\t" + download_file_path)
# with open(download_file_path, "wb") as download_file:
# download_file.write(src_blob.download_blob().readall())
try:
action = self.tgt.start_copy_from_url(src_blob.url)
copy_id = action["copy_id"]
status = action["copy_status"]
error = action["error_code"]
self.wait_condition(condition=self.check_copy_status
, timeout=self.max_wait_in_sec
, granularity=self.recheck)
properties = self.tgt.get_blob_properties()
print("Total bytes: " + str(properties.size))
copy_props = properties.copy
if copy_props["status"] != "success":
self.tgt.abort_copy(copy_id=copy_props["id"])
print(f"Unable to copy blob %s to %s. Status: %s" % (src_file, tgt_file, copy_props.status))
result = messages.message["copy_files_failed"]
break
# Note: We do not release the lease in case of errors
if source_props.lease.state == "leased":
# Break the lease on the source blob.
lease.break_lease()
# Update the source blob's properties to check the lease state.
source_props = src_blob.get_blob_properties()
print("Lease state: " + source_props.lease.state)
result = messages.message["ok"]
except exceptions.ResourceNotFoundError as e:
print("Azure reported a resource not found error: ", e)
result = messages.message["resource_not_found"]
result["reference"] = f"source: %s, targte: %s" % (src_file, tgt_file)
return result
