async def test_rename_file_with_file_sas_async(self, datalake_storage_account_name, datalake_storage_account_key): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) # SAS URL is calculated from storage key, so this test runs live only token = generate_file_sas(self.dsc.account_name, self.file_system_name, None, "oldfile", datalake_storage_account_key, permission=FileSasPermissions(read=True, create=True, write=True, delete=True, move=True), expiry=datetime.utcnow() + timedelta(hours=1), ) new_token = generate_file_sas(self.dsc.account_name, self.file_system_name, None, "newname", datalake_storage_account_key, permission=FileSasPermissions(read=True, create=True, write=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # read the created file which is under root directory file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, "oldfile", credential=token) await file_client.create_file() data_bytes = b"abc" await file_client.append_data(data_bytes, 0, 3) await file_client.flush_data(3) new_client = await file_client.rename_file(file_client.file_system_name+'/'+'newname'+'?'+new_token) data = await (await new_client.download_file()).readall() self.assertEqual(data, data_bytes) self.assertEqual(new_client.path_name, "newname")
async def _test_rename_file_with_file_sas(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return token = generate_file_sas(self.dsc.account_name, self.file_system_name, None, "oldfile", self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY, permission=FileSasPermissions(read=True, create=True, write=True, delete=True, move=True), expiry=datetime.utcnow() + timedelta(hours=1), ) new_token = generate_file_sas(self.dsc.account_name, self.file_system_name, None, "newname", self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY, permission=FileSasPermissions(read=True, create=True, write=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # read the created file which is under root directory file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, "oldfile", credential=token) await file_client.create_file() data_bytes = b"abc" await file_client.append_data(data_bytes, 0, 3) await file_client.flush_data(3) new_client = await file_client.rename_file(file_client.file_system_name+'/'+'newname'+'?'+new_token) data = await (await new_client.download_file()).readall() self.assertEqual(data, data_bytes) self.assertEqual(new_client.path_name, "newname")
async def _test_read_file_with_user_delegation_key(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Create file file_client = await self._create_file_and_return_client() data = self.get_random_bytes(1024) # Upload data to file await file_client.append_data(data, 0, len(data)) await file_client.flush_data(len(data)) # Get user delegation key token_credential = self.generate_async_oauth_token() service_client = DataLakeServiceClient(self._get_oauth_account_url(), credential=token_credential) user_delegation_key = await service_client.get_user_delegation_key(datetime.utcnow(), datetime.utcnow() + timedelta(hours=1)) sas_token = generate_file_sas(file_client.account_name, file_client.file_system_name, None, file_client.path_name, user_delegation_key=user_delegation_key, permission=FileSasPermissions(read=True, create=True, write=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # doanload the data and make sure it is the same as uploaded data new_file_client = DataLakeFileClient(self._get_account_url(), file_client.file_system_name, file_client.path_name, credential=sas_token) downloaded_data = await new_file_client.read_file() self.assertEqual(data, downloaded_data)
def test_file_sas_only_applies_to_file_level(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # SAS URL is calculated from storage key, so this test runs live only file_name = self._get_file_reference() directory_name = self._get_directory_reference() self._create_file_and_return_client(directory=directory_name, file=file_name) # generate a token with file level read and write permissions token = generate_file_sas( self.dsc.account_name, self.file_system_name, directory_name, file_name, self.dsc.credential.account_key, permission=FileSasPermissions(read=True, write=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # read the created file which is under root directory file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, directory_name + '/' + file_name, credential=token) properties = file_client.get_file_properties() # make sure we can read the file properties self.assertIsNotNone(properties) # try to write to the created file with the token response = file_client.append_data(b"abcd", 0, 4, validate_content=True) self.assertIsNotNone(response) # the token is for file level, so users are not supposed to have access to file system level operations file_system_client = FileSystemClient(self.dsc.url, self.file_system_name, credential=token) with self.assertRaises(ClientAuthenticationError): file_system_client.get_file_system_properties() # the token is for file level, so users are not supposed to have access to directory level operations directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, credential=token) with self.assertRaises(ClientAuthenticationError): directory_client.get_directory_properties()
def test_preauthorize_user_with_user_delegation_key( self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # SAS URL is calculated from storage key, so this test runs live only # Create file file_client = self._create_file_and_return_client() data = self.get_random_bytes(1024) # Upload data to file file_client.append_data(data, 0, len(data)) file_client.flush_data(len(data)) file_client.set_access_control( owner="68390a19-a643-458b-b726-408abf67b4fc", permissions='0777') acl = file_client.get_access_control() # Get user delegation key token_credential = self.generate_oauth_token() service_client = DataLakeServiceClient( self._get_account_url(datalake_storage_account_name), credential=token_credential) user_delegation_key = service_client.get_user_delegation_key( datetime.utcnow(), datetime.utcnow() + timedelta(hours=1)) sas_token = generate_file_sas( file_client.account_name, file_client.file_system_name, None, file_client.path_name, user_delegation_key, permission=FileSasPermissions(read=True, write=True, manage_access_control=True, manage_ownership=True), expiry=datetime.utcnow() + timedelta(hours=1), preauthorized_agent_object_id="68390a19-a643-458b-b726-408abf67b4fc" ) # doanload the data and make sure it is the same as uploaded data new_file_client = DataLakeFileClient( self._get_account_url(datalake_storage_account_name), file_client.file_system_name, file_client.path_name, credential=sas_token) acl = new_file_client.set_access_control(permissions='0777') self.assertIsNotNone(acl)
def test_set_acl_with_user_delegation_key(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # SAS URL is calculated from storage key, so this test runs live only # Create file file_client = self._create_file_and_return_client() data = self.get_random_bytes(1024) # Upload data to file file_client.append_data(data, 0, len(data)) file_client.flush_data(len(data)) # Get user delegation key token_credential = self.generate_oauth_token() service_client = DataLakeServiceClient( self._get_account_url(datalake_storage_account_name), credential=token_credential) user_delegation_key = service_client.get_user_delegation_key( datetime.utcnow(), datetime.utcnow() + timedelta(hours=1)) sas_token = generate_file_sas( file_client.account_name, file_client.file_system_name, None, file_client.path_name, user_delegation_key, permission=FileSasPermissions(execute=True, manage_access_control=True, manage_ownership=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # doanload the data and make sure it is the same as uploaded data new_file_client = DataLakeFileClient( self._get_account_url(datalake_storage_account_name), file_client.file_system_name, file_client.path_name, credential=sas_token) acl = 'user::rwx,group::r-x,other::rwx' owner = "dc140949-53b7-44af-b1e9-cd994951fb86" new_file_client.set_access_control(acl=acl, owner=owner) access_control = new_file_client.get_access_control() self.assertEqual(acl, access_control['acl']) self.assertEqual(owner, access_control['owner'])
def test_read_file_with_user_delegation_key(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # SAS URL is calculated from storage key, so this test runs live only # Create file file_client = self._create_file_and_return_client() data = self.get_random_bytes(1024) # Upload data to file file_client.append_data(data, 0, len(data)) file_client.flush_data(len(data)) # Get user delegation key token_credential = self.generate_oauth_token() service_client = DataLakeServiceClient( self._get_account_url(datalake_storage_account_name), credential=token_credential, logging_enable=True) user_delegation_key = service_client.get_user_delegation_key( datetime.utcnow(), datetime.utcnow() + timedelta(hours=1)) sas_token = generate_file_sas( file_client.account_name, file_client.file_system_name, None, file_client.path_name, user_delegation_key, permission=FileSasPermissions(read=True, create=True, write=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # doanload the data and make sure it is the same as uploaded data new_file_client = DataLakeFileClient( self._get_account_url(datalake_storage_account_name), file_client.file_system_name, file_client.path_name, credential=sas_token, logging_enable=True) downloaded_data = new_file_client.download_file().readall() self.assertEqual(data, downloaded_data)
def main(event: func.EventGridEvent): result = json.dumps({ 'id': event.id, 'data': event.get_json(), 'topic': event.topic, 'subject': event.subject, 'event_type': event.event_type, }) logging.info('Python EventGrid trigger processed an event: %s', result) blob_url = event.get_json().get('url') logging.info('blob URL: %s', blob_url) blob_name = blob_url.split("/")[-1].split("?")[0] logging.info('blob name: %s', blob_name) origin_container_name = blob_url.split("/")[-2].split("?")[0] logging.info('container name: %s', origin_container_name) storage_account_name = blob_url.split("//")[1].split(".")[0] logging.info('storage account name: %s', storage_account_name) ams_account_name = os.getenv('ACCOUNTNAME') resource_group_name = os.getenv('RESOURCEGROUP') subscription_id = os.getenv('SUBSCRIPTIONID') client_id = os.getenv('AZURE_CLIENT_ID') client_secret = os.getenv('AZURE_CLIENT_SECRET') TENANT_ID = os.getenv('AZURE_TENANT_ID') storage_blob_url = 'https://' + storage_account_name + '.blob.core.windows.net/' transform_name = 'faceredact' LOGIN_ENDPOINT = AZURE_PUBLIC_CLOUD.endpoints.active_directory RESOURCE = AZURE_PUBLIC_CLOUD.endpoints.active_directory_resource_id logging.info('login_endpoint: %s', LOGIN_ENDPOINT) logging.info('tenant_id: %s', TENANT_ID) out_asset_name = 'faceblurringOutput_' + datetime.utcnow().strftime( "%m-%d-%Y_%H:%M:%S") out_alternate_id = 'faceblurringOutput_' + datetime.utcnow().strftime( "%m-%d-%Y_%H:%M:%S") out_description = 'Redacted video with blurred faces' context = adal.AuthenticationContext(LOGIN_ENDPOINT + "/" + TENANT_ID) credentials = AdalAuthentication( context.acquire_token_with_client_credentials, RESOURCE, client_id, client_secret) client = AzureMediaServices(credentials, subscription_id) output_asset = Asset(alternate_id=out_alternate_id, description=out_description) client.assets.create_or_update(resource_group_name, ams_account_name, out_asset_name, output_asset) token_credential = DefaultAzureCredential() datalake_service_client = DataLakeServiceClient( account_url=storage_blob_url, credential=token_credential) delegation_key = datalake_service_client.get_user_delegation_key( key_start_time=datetime.utcnow(), key_expiry_time=datetime.utcnow() + timedelta(hours=1)) sas_token = generate_file_sas(account_name=storage_account_name, file_system_name=origin_container_name, directory_name="", file_name=blob_name, credential=delegation_key, permission=FileSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), protocol="https") sas_url = "{}?{}".format(blob_url, sas_token) logging.info(sas_url) job_name = 'Faceblurring-job_' + datetime.utcnow().strftime( "%m-%d-%Y_%H:%M:%S") job_input = JobInputHttp(label="Video_asset", files=[sas_url]) job_output = JobOutputAsset(asset_name=out_asset_name) job_parameters = Job(input=job_input, outputs=[job_output]) client.jobs.create(resource_group_name, ams_account_name, transform_name, job_name, parameters=job_parameters)