def filesystem_list(self
, prefix = None
, continuation = None
, maxResults = None
, timeout = None
):
"""
List filesystems and their properties in given account.
https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/filesystem/list
Basic variant:
GET https://{accountName}.{dnsSuffix}/?resource=account
With optional parameters:
GET https://{accountName}.{dnsSuffix}/?resource=account&prefix={prefix}&continuation={continuation}&maxResults={maxResults}&timeout={timeout}
"""
url = 'https://{storage_account_name}.{azure_datalake_dns_suffix}/'.format(
storage_account_name = self.__storage_account_name
, azure_datalake_dns_suffix = self.__azure_datalake_dns_suffix
)
sas_token = self.__account_sas_generator.generate_account(
services = Services.BLOB
, resource_types = ResourceTypes.SERVICE
, permission=AccountPermissions(list=True)
, expiry=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(minutes=2)
, start=None
, ip=None
, protocol=Protocol.HTTPS
)
# Create the params of the query from the sas_token
params = parse_qs(sas_token)
# Add specific params for this operation
params['resource']='account'
if not prefix is None:
params['prefix']=prefix
if not continuation is None:
params['continuation']=continuation
if not maxResults is None:
params['maxResults']=maxResults
if not timeout is None:
params['timeout']=timeout
# Execute the request
response = requests.get(url, params=params)
# Raise an error if the response code is not a positive one
response.raise_for_status()
return json.loads(response.text)
def path_read(self
, filesystem
, path
, timeout = None
, request_headers = None
):
"""
Read the contents of a file. For read operations, range requests are supported.
This operation supports conditional HTTP requests.
https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/read
Basic variant:
GET https://{accountName}.{dnsSuffix}/{filesystem}/{path}
With optional parameters:
GET https://{accountName}.{dnsSuffix}/{filesystem}/{path}?timeout={timeout}
"""
url = 'https://{storage_account_name}.{azure_datalake_dns_suffix}/{filesystem}/{path}'.format(
storage_account_name = self.__storage_account_name
, azure_datalake_dns_suffix = self.__azure_datalake_dns_suffix
, filesystem = filesystem
, path = path
)
sas_token = self.__account_sas_generator.generate_account(
services = Services.BLOB
, resource_types = ResourceTypes.OBJECT
, permission=AccountPermissions(read=True)
, expiry=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(minutes=2)
, start=None
, ip=None
, protocol=Protocol.HTTPS
)
# Create the params of the query from the sas_token
params = parse_qs(sas_token)
# Add specific params for this operation
if not timeout is None:
params['timeout']=timeout
# Execute the request
response = requests.get(url, params=params, headers=request_headers)
# Raise an error if the response code is not a positive one
response.raise_for_status()
if response.headers['Content-Type'] == 'text/plain':
return response.text
elif response.headers['Content-Type'] == 'application/json':
return json.loads(response.text)
elif response.headers['Content-Type'] == 'application/octet-stream':
raise NotImplementedError()
else:
raise TypeError('The returned response has an unknown content type: [{}]'.format(response.headers['Content-Type']))
def filesystem_create(self
, filesystem
, timeout = None
):
"""
Create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails.
This operation does not support conditional HTTP requests.
https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/filesystem/create
Basic variant:
PUT http://{accountName}.{dnsSuffix}/{filesystem}?resource=filesystem
With optional parameters:
PUT http://{accountName}.{dnsSuffix}/{filesystem}?resource=filesystem&timeout={timeout}
"""
url = 'https://{storage_account_name}.{azure_datalake_dns_suffix}/{filesystem}'.format(
storage_account_name = self.__storage_account_name
, azure_datalake_dns_suffix = self.__azure_datalake_dns_suffix
, filesystem=filesystem
)
sas_token = self.__account_sas_generator.generate_account(
services = Services.BLOB
, resource_types = ResourceTypes.CONTAINER
, permission=AccountPermissions(create=True)
, expiry=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(minutes=2)
, start=None
, ip=None
, protocol=Protocol.HTTPS
)
# Create the params of the query from the sas_token
params = parse_qs(sas_token)
# Add specific params for this operation
params['resource']='filesystem'
if not timeout is None:
params['timeout']=timeout
# Execute the request
response = requests.put(url, params=params)
# Raise an error if the response code is not a positive one
response.raise_for_status()
return True
def filesystem_get_properties(self
, filesystem
, timeout = None
):
"""
All system and user-defined filesystem properties are specified in the response headers.
https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/filesystem/getproperties
Basic variant:
HEAD https://{accountName}.{dnsSuffix}/{filesystem}?resource=filesystem
With optional parameters:
HEAD https://{accountName}.{dnsSuffix}/{filesystem}?resource=filesystem&timeout={timeout}
"""
url = 'https://{storage_account_name}.{azure_datalake_dns_suffix}/{filesystem}'.format(
storage_account_name = self.__storage_account_name
, azure_datalake_dns_suffix = self.__azure_datalake_dns_suffix
, filesystem=filesystem
)
sas_token = self.__account_sas_generator.generate_account(
services = Services.BLOB
, resource_types = ResourceTypes.CONTAINER
, permission=AccountPermissions(read=True)
, expiry=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(minutes=2)
, start=None
, ip=None
, protocol=Protocol.HTTPS
)
# Create the params of the query from the sas_token
params = parse_qs(sas_token)
# Add specific params for this operation
params['resource']='filesystem'
if not timeout is None:
params['timeout']=timeout
# Execute the request
response = requests.head(url, params=params)
# Raise an error if the response code is not a positive one
response.raise_for_status()
return response.headers
def path_update(self
, filesystem
, path
, action
, position = None
, retainUncommittedData = None
, close = None
, timeout = None
, request_headers = None
, data_to_append = None
):
"""
Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file,
sets properties for a file or directory, or sets access control for a file or directory.
Data can only be appended to a file.
This operation supports conditional HTTP requests.
https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
Basic variant:
PATCH http://{accountName}.{dnsSuffix}/{filesystem}/{path}?action={action}
With optional parameters:
PATCH http://{accountName}.{dnsSuffix}/{filesystem}/{path}?action={action}&position={position}&retainUncommittedData={retainUncommittedData}&close={close}&timeout={timeout}
"""
url = 'https://{storage_account_name}.{azure_datalake_dns_suffix}/{filesystem}/{path}'.format(
storage_account_name = self.__storage_account_name
, azure_datalake_dns_suffix = self.__azure_datalake_dns_suffix
, filesystem = filesystem
, path = path
)
sas_token = self.__account_sas_generator.generate_account(
services = Services.BLOB
, resource_types = ResourceTypes.OBJECT
, permission=AccountPermissions(write=True)
, expiry=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(minutes=2)
, start=None
, ip=None
, protocol=Protocol.HTTPS
)
# Create the params of the query from the sas_token
params = parse_qs(sas_token)
# Add specific params for this operation
params['action']=action
if not position is None:
params['position']=position
if not retainUncommittedData is None:
params['retainUncommittedData']=retainUncommittedData
if not close is None:
params['close']=close
if not timeout is None:
params['timeout']=timeout
# Execute the request
response = requests.patch(url, params=params, headers=request_headers, data=data_to_append)
# Raise an error if the response code is not a positive one
response.raise_for_status()
return response.headers
def path_list(self
, filesystem
, recursive
, directory = None
, continuation = None
, maxResults = None
, upn = None
, timeout = None
, request_headers = None
):
"""
List filesystem paths and their properties.
https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/list
Basic variant:
GET https://{accountName}.{dnsSuffix}/{filesystem}?recursive={recursive}&resource=filesystem
With optional parameters:
GET https://{accountName}.{dnsSuffix}/{filesystem}?directory={directory}&recursive={recursive}&continuation={continuation}&maxResults={maxResults}&upn={upn}&resource=filesystem&timeout={timeout}
"""
url = 'https://{storage_account_name}.{azure_datalake_dns_suffix}/{filesystem}'.format(
storage_account_name = self.__storage_account_name
, azure_datalake_dns_suffix = self.__azure_datalake_dns_suffix
, filesystem = filesystem
)
sas_token = self.__account_sas_generator.generate_account(
services = Services.BLOB
, resource_types = ResourceTypes.CONTAINER
, permission=AccountPermissions(list=True)
, expiry=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(minutes=2)
, start=None
, ip=None
, protocol=Protocol.HTTPS
)
# Create the params of the query from the sas_token
params = parse_qs(sas_token)
# Add specific params for this operation
# We convert `recursive` to str just in case it's boolean,
# and we lower it in case we pass 'True' or 'FALSE'.
params['resource']='filesystem'
params['recursive']=str(recursive).lower()
if not directory is None:
params['directory']=directory
if not continuation is None:
params['continuation']=continuation
if not maxResults is None:
params['maxResults']=maxResults
if not upn is None:
params['upn']=upn
if not timeout is None:
params['timeout']=timeout
# Execute the request
response = requests.get(url, params=params, headers=request_headers)
# Raise an error if the response code is not a positive one
response.raise_for_status()
return response
def path_get_properties(self
, filesystem
, path
, action = None
, upn = None
, timeout = None
, request_headers = None
):
"""
Get Properties returns all system and user defined properties for a path.
Get Status returns all system defined properties for a path.
Get Access Control List returns the access control list for a path.
This operation supports conditional HTTP requests.
https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/getproperties
Basic variant:
HEAD https://{accountName}.{dnsSuffix}/{filesystem}/{path}
With optional parameters:
HEAD https://{accountName}.{dnsSuffix}/{filesystem}/{path}?action={action}&upn={upn}&timeout={timeout}
"""
if path is None:
raise ValueError('The parameter [path] cannot be None.')
url = 'https://{storage_account_name}.{azure_datalake_dns_suffix}/{filesystem}/{path}'.format(
storage_account_name = self.__storage_account_name
, azure_datalake_dns_suffix = self.__azure_datalake_dns_suffix
, filesystem = filesystem
, path = path
)
sas_token = self.__account_sas_generator.generate_account(
services = Services.BLOB
, resource_types = ResourceTypes.OBJECT
, permission=AccountPermissions(read=True)
, expiry=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(minutes=2)
, start=None
, ip=None
, protocol=Protocol.HTTPS
)
# Create the params of the query from the sas_token
params = parse_qs(sas_token)
# Add specific params for this operation
# We convert `recursive` to str just in case it's boolean,
# and we lower it in case we pass 'True' or 'FALSE'.
if not action is None:
params['action']=action
if not upn is None:
params['upn']=str(upn).lower()
if not timeout is None:
params['timeout']=timeout
# Execute the request
response = requests.head(url, params=params, headers=request_headers)
# Raise an error if the response code is not a positive one
response.raise_for_status()
return dict(response.headers)
def path_create(self
, filesystem
, path
, resource = None
, continuation = None
, mode = None
, timeout = None
, request_headers = None
):
"""
Use for: Create File | Create Directory | Rename File | Rename Directory
Create or rename a file or directory. By default, the destination is overwritten
and if the destination already exists and has a lease the lease is broken.
This operation supports conditional HTTP requests.
To fail if the destination already exists, use a conditional request with If-None-Match: "*".
https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create
Basic variant:
PUT https://{accountName}.{dnsSuffix}/{filesystem}/{path}
With optional parameters:
PUT https://{accountName}.{dnsSuffix}/{filesystem}/{path}?resource={resource}&continuation={
"""
url = 'https://{storage_account_name}.{azure_datalake_dns_suffix}/{filesystem}/{path}'.format(
storage_account_name = self.__storage_account_name
, azure_datalake_dns_suffix = self.__azure_datalake_dns_suffix
, filesystem = filesystem
, path = path
)
sas_token = self.__account_sas_generator.generate_account(
services = Services.BLOB
, resource_types = ResourceTypes.OBJECT
, permission=AccountPermissions(write=True)
, expiry=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(minutes=2)
, start=None
, ip=None
, protocol=Protocol.HTTPS
)
# Create the params of the query from the sas_token
params = parse_qs(sas_token)
# Add specific params for this operation
if not resource is None:
params['resource']=resource
if not continuation is None:
params['continuation']=continuation
if not mode is None:
params['mode']=mode
if not timeout is None:
params['timeout']=timeout
# Execute the request
response = requests.put(url, params=params, headers=request_headers)
# Raise an error if the response code is not a positive one
response.raise_for_status()
return response.headers