async def test_rename_file_with_file_sas_async(self, datalake_storage_account_name, datalake_storage_account_key):
await self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# SAS URL is calculated from storage key, so this test runs live only
token = generate_file_sas(self.dsc.account_name,
self.file_system_name,
None,
"oldfile",
datalake_storage_account_key,
permission=FileSasPermissions(read=True, create=True, write=True, delete=True, move=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
new_token = generate_file_sas(self.dsc.account_name,
self.file_system_name,
None,
"newname",
datalake_storage_account_key,
permission=FileSasPermissions(read=True, create=True, write=True, delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# read the created file which is under root directory
file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, "oldfile", credential=token)
await file_client.create_file()
data_bytes = b"abc"
await file_client.append_data(data_bytes, 0, 3)
await file_client.flush_data(3)
new_client = await file_client.rename_file(file_client.file_system_name+'/'+'newname'+'?'+new_token)
data = await (await new_client.download_file()).readall()
self.assertEqual(data, data_bytes)
self.assertEqual(new_client.path_name, "newname")
async def _test_rename_file_with_file_sas(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
token = generate_file_sas(self.dsc.account_name,
self.file_system_name,
None,
"oldfile",
self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY,
permission=FileSasPermissions(read=True, create=True, write=True, delete=True, move=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
new_token = generate_file_sas(self.dsc.account_name,
self.file_system_name,
None,
"newname",
self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY,
permission=FileSasPermissions(read=True, create=True, write=True, delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# read the created file which is under root directory
file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, "oldfile", credential=token)
await file_client.create_file()
data_bytes = b"abc"
await file_client.append_data(data_bytes, 0, 3)
await file_client.flush_data(3)
new_client = await file_client.rename_file(file_client.file_system_name+'/'+'newname'+'?'+new_token)
data = await (await new_client.download_file()).readall()
self.assertEqual(data, data_bytes)
self.assertEqual(new_client.path_name, "newname")
async def _test_read_file_with_user_delegation_key(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Create file
file_client = await self._create_file_and_return_client()
data = self.get_random_bytes(1024)
# Upload data to file
await file_client.append_data(data, 0, len(data))
await file_client.flush_data(len(data))
# Get user delegation key
token_credential = self.generate_async_oauth_token()
service_client = DataLakeServiceClient(self._get_oauth_account_url(), credential=token_credential)
user_delegation_key = await service_client.get_user_delegation_key(datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
sas_token = generate_file_sas(file_client.account_name,
file_client.file_system_name,
None,
file_client.path_name,
user_delegation_key=user_delegation_key,
permission=FileSasPermissions(read=True, create=True, write=True, delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# doanload the data and make sure it is the same as uploaded data
new_file_client = DataLakeFileClient(self._get_account_url(),
file_client.file_system_name,
file_client.path_name,
credential=sas_token)
downloaded_data = await new_file_client.read_file()
self.assertEqual(data, downloaded_data)
def test_file_sas_only_applies_to_file_level(self,
datalake_storage_account_name,
datalake_storage_account_key):
self._setUp(datalake_storage_account_name,
datalake_storage_account_key)
# SAS URL is calculated from storage key, so this test runs live only
file_name = self._get_file_reference()
directory_name = self._get_directory_reference()
self._create_file_and_return_client(directory=directory_name,
file=file_name)
# generate a token with file level read and write permissions
token = generate_file_sas(
self.dsc.account_name,
self.file_system_name,
directory_name,
file_name,
self.dsc.credential.account_key,
permission=FileSasPermissions(read=True, write=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# read the created file which is under root directory
file_client = DataLakeFileClient(self.dsc.url,
self.file_system_name,
directory_name + '/' + file_name,
credential=token)
properties = file_client.get_file_properties()
# make sure we can read the file properties
self.assertIsNotNone(properties)
# try to write to the created file with the token
response = file_client.append_data(b"abcd",
0,
4,
validate_content=True)
self.assertIsNotNone(response)
# the token is for file level, so users are not supposed to have access to file system level operations
file_system_client = FileSystemClient(self.dsc.url,
self.file_system_name,
credential=token)
with self.assertRaises(ClientAuthenticationError):
file_system_client.get_file_system_properties()
# the token is for file level, so users are not supposed to have access to directory level operations
directory_client = DataLakeDirectoryClient(self.dsc.url,
self.file_system_name,
directory_name,
credential=token)
with self.assertRaises(ClientAuthenticationError):
directory_client.get_directory_properties()
def test_preauthorize_user_with_user_delegation_key(
self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name,
datalake_storage_account_key)
# SAS URL is calculated from storage key, so this test runs live only
# Create file
file_client = self._create_file_and_return_client()
data = self.get_random_bytes(1024)
# Upload data to file
file_client.append_data(data, 0, len(data))
file_client.flush_data(len(data))
file_client.set_access_control(
owner="68390a19-a643-458b-b726-408abf67b4fc", permissions='0777')
acl = file_client.get_access_control()
# Get user delegation key
token_credential = self.generate_oauth_token()
service_client = DataLakeServiceClient(
self._get_account_url(datalake_storage_account_name),
credential=token_credential)
user_delegation_key = service_client.get_user_delegation_key(
datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
sas_token = generate_file_sas(
file_client.account_name,
file_client.file_system_name,
None,
file_client.path_name,
user_delegation_key,
permission=FileSasPermissions(read=True,
write=True,
manage_access_control=True,
manage_ownership=True),
expiry=datetime.utcnow() + timedelta(hours=1),
preauthorized_agent_object_id="68390a19-a643-458b-b726-408abf67b4fc"
)
# doanload the data and make sure it is the same as uploaded data
new_file_client = DataLakeFileClient(
self._get_account_url(datalake_storage_account_name),
file_client.file_system_name,
file_client.path_name,
credential=sas_token)
acl = new_file_client.set_access_control(permissions='0777')
self.assertIsNotNone(acl)
def test_set_acl_with_user_delegation_key(self,
datalake_storage_account_name,
datalake_storage_account_key):
self._setUp(datalake_storage_account_name,
datalake_storage_account_key)
# SAS URL is calculated from storage key, so this test runs live only
# Create file
file_client = self._create_file_and_return_client()
data = self.get_random_bytes(1024)
# Upload data to file
file_client.append_data(data, 0, len(data))
file_client.flush_data(len(data))
# Get user delegation key
token_credential = self.generate_oauth_token()
service_client = DataLakeServiceClient(
self._get_account_url(datalake_storage_account_name),
credential=token_credential)
user_delegation_key = service_client.get_user_delegation_key(
datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
sas_token = generate_file_sas(
file_client.account_name,
file_client.file_system_name,
None,
file_client.path_name,
user_delegation_key,
permission=FileSasPermissions(execute=True,
manage_access_control=True,
manage_ownership=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# doanload the data and make sure it is the same as uploaded data
new_file_client = DataLakeFileClient(
self._get_account_url(datalake_storage_account_name),
file_client.file_system_name,
file_client.path_name,
credential=sas_token)
acl = 'user::rwx,group::r-x,other::rwx'
owner = "dc140949-53b7-44af-b1e9-cd994951fb86"
new_file_client.set_access_control(acl=acl, owner=owner)
access_control = new_file_client.get_access_control()
self.assertEqual(acl, access_control['acl'])
self.assertEqual(owner, access_control['owner'])
def test_read_file_with_user_delegation_key(self,
datalake_storage_account_name,
datalake_storage_account_key):
self._setUp(datalake_storage_account_name,
datalake_storage_account_key)
# SAS URL is calculated from storage key, so this test runs live only
# Create file
file_client = self._create_file_and_return_client()
data = self.get_random_bytes(1024)
# Upload data to file
file_client.append_data(data, 0, len(data))
file_client.flush_data(len(data))
# Get user delegation key
token_credential = self.generate_oauth_token()
service_client = DataLakeServiceClient(
self._get_account_url(datalake_storage_account_name),
credential=token_credential,
logging_enable=True)
user_delegation_key = service_client.get_user_delegation_key(
datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
sas_token = generate_file_sas(
file_client.account_name,
file_client.file_system_name,
None,
file_client.path_name,
user_delegation_key,
permission=FileSasPermissions(read=True,
create=True,
write=True,
delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# doanload the data and make sure it is the same as uploaded data
new_file_client = DataLakeFileClient(
self._get_account_url(datalake_storage_account_name),
file_client.file_system_name,
file_client.path_name,
credential=sas_token,
logging_enable=True)
downloaded_data = new_file_client.download_file().readall()
self.assertEqual(data, downloaded_data)
def main(event: func.EventGridEvent):
result = json.dumps({
'id': event.id,
'data': event.get_json(),
'topic': event.topic,
'subject': event.subject,
'event_type': event.event_type,
})
logging.info('Python EventGrid trigger processed an event: %s', result)
blob_url = event.get_json().get('url')
logging.info('blob URL: %s', blob_url)
blob_name = blob_url.split("/")[-1].split("?")[0]
logging.info('blob name: %s', blob_name)
origin_container_name = blob_url.split("/")[-2].split("?")[0]
logging.info('container name: %s', origin_container_name)
storage_account_name = blob_url.split("//")[1].split(".")[0]
logging.info('storage account name: %s', storage_account_name)
ams_account_name = os.getenv('ACCOUNTNAME')
resource_group_name = os.getenv('RESOURCEGROUP')
subscription_id = os.getenv('SUBSCRIPTIONID')
client_id = os.getenv('AZURE_CLIENT_ID')
client_secret = os.getenv('AZURE_CLIENT_SECRET')
TENANT_ID = os.getenv('AZURE_TENANT_ID')
storage_blob_url = 'https://' + storage_account_name + '.blob.core.windows.net/'
transform_name = 'faceredact'
LOGIN_ENDPOINT = AZURE_PUBLIC_CLOUD.endpoints.active_directory
RESOURCE = AZURE_PUBLIC_CLOUD.endpoints.active_directory_resource_id
logging.info('login_endpoint: %s', LOGIN_ENDPOINT)
logging.info('tenant_id: %s', TENANT_ID)
out_asset_name = 'faceblurringOutput_' + datetime.utcnow().strftime(
"%m-%d-%Y_%H:%M:%S")
out_alternate_id = 'faceblurringOutput_' + datetime.utcnow().strftime(
"%m-%d-%Y_%H:%M:%S")
out_description = 'Redacted video with blurred faces'
context = adal.AuthenticationContext(LOGIN_ENDPOINT + "/" + TENANT_ID)
credentials = AdalAuthentication(
context.acquire_token_with_client_credentials, RESOURCE, client_id,
client_secret)
client = AzureMediaServices(credentials, subscription_id)
output_asset = Asset(alternate_id=out_alternate_id,
description=out_description)
client.assets.create_or_update(resource_group_name, ams_account_name,
out_asset_name, output_asset)
token_credential = DefaultAzureCredential()
datalake_service_client = DataLakeServiceClient(
account_url=storage_blob_url, credential=token_credential)
delegation_key = datalake_service_client.get_user_delegation_key(
key_start_time=datetime.utcnow(),
key_expiry_time=datetime.utcnow() + timedelta(hours=1))
sas_token = generate_file_sas(account_name=storage_account_name,
file_system_name=origin_container_name,
directory_name="",
file_name=blob_name,
credential=delegation_key,
permission=FileSasPermissions(read=True),
expiry=datetime.utcnow() +
timedelta(hours=1),
protocol="https")
sas_url = "{}?{}".format(blob_url, sas_token)
logging.info(sas_url)
job_name = 'Faceblurring-job_' + datetime.utcnow().strftime(
"%m-%d-%Y_%H:%M:%S")
job_input = JobInputHttp(label="Video_asset", files=[sas_url])
job_output = JobOutputAsset(asset_name=out_asset_name)
job_parameters = Job(input=job_input, outputs=[job_output])
client.jobs.create(resource_group_name,
ams_account_name,
transform_name,
job_name,
parameters=job_parameters)
