def move_blobs_to_processing(**context): results = blob_service.connection.list_blobs( input_container, processing_file_prefix) blobs_moved = 0 blob_urls = [] for blob in results: print("\t Blob name: " + blob.name) # Generate a SAS token for blob access blob_input_url = blob_service.connection.make_blob_url( input_container, blob.name, sas_token=blob_service.connection.generate_blob_shared_access_signature( input_container, blob.name, permission=BlobPermissions(read=True), expiry=datetime.utcnow() + timedelta(days=5))) print("\t SAS URL:{}".format(blob_input_url)) # Copy blob to processing bucket blob_service.connection.copy_blob( output_container, blob.name, blob_input_url, requires_sync=True) # Generate a SAS token the now moved blob for downstream dags blob_output_url = blob_service.connection.make_blob_url( output_container, blob.name, sas_token=blob_service.connection.generate_blob_shared_access_signature( output_container, blob.name, permission=BlobPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1))) blobs_moved += 1 blob_urls.append(blob_output_url) def trigger_processing_dag(context, dag_run_obj): dag_run_obj.payload = { "image_url": blob_output_url, } return dag_run_obj TriggerDagRunOperator( task_id="trigger_processing", trigger_dag_id="image_processing", python_callable=trigger_processing_dag, dag=dag ).execute(context) # Remove existing blob blob_service.connection.delete_blob(input_container, blob.name) return blob_urls
def _get_files_from_bfs(cli_ctx, bfs, path, expiry): """Returns a list of files and directories under given path on mounted blob container. :param models.AzureBlobFileSystemReference bfs: blob file system reference. :param str path: path to list files from. :param int expiry: SAS expiration time in minutes. """ from azure.storage.blob import BlockBlobService from azure.storage.blob.models import Blob, BlobPermissions result = [] service = BlockBlobService(bfs.account_name, _get_storage_account_key(cli_ctx, bfs.account_name, None)) effective_path = _get_path_for_storage(path) folders = set() for b in service.list_blobs(bfs.container_name, effective_path + '/', delimiter='/'): if isinstance(b, Blob): name = os.path.basename(b.name) sas = service.generate_blob_shared_access_signature( bfs.container_name, b.name, BlobPermissions(read=True), expiry=datetime.datetime.utcnow() + datetime.timedelta(minutes=expiry)) result.append( LogFile( name, service.make_blob_url(bfs.container_name, b.name, 'https', sas), False, b.properties.content_length)) else: name = b.name.split('/')[-2] folders.add(name) result.append(LogFile(name, None, True, None)) result = [f for f in result if f.is_directory or f.name not in folders] return result
def _create_short_lived_file_sas(account_name, account_key, share, directory_name, file_name): # if dir is empty string change it to None directory_name = directory_name if directory_name else None expiry = (datetime.utcnow() + timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') sas = SharedAccessSignature(account_name, account_key) return sas.generate_file(share, directory_name=directory_name, file_name=file_name, permission=BlobPermissions(read=True), expiry=expiry, protocol='https')
def _create_short_lived_blob_sas(account_name, account_key, container, blob): expiry = (datetime.utcnow() + timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') sas = SharedAccessSignature(account_name, account_key) return sas.generate_blob(container, blob, permission=BlobPermissions(read=True), expiry=expiry, protocol='https')
def create_short_lived_container_sas(account_name, account_key, container): from datetime import datetime, timedelta from azure.storage.sharedaccesssignature import SharedAccessSignature from azure.storage.blob.models import BlobPermissions expiry = (datetime.utcnow() + timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') sas = SharedAccessSignature(account_name, account_key) return sas.generate_container(container, permission=BlobPermissions(read=True), expiry=expiry, protocol='https')
def get_sas_url(name): baseblobservice = BaseBlobService(account_name='', account_key='') sasToken = baseblobservice.generate_blob_shared_access_signature( container_name='images', blob_name=name, permission=BlobPermissions(read=True, write=True, create=True), start=datetime.now() - timedelta(hours=2), expiry=datetime.now() + timedelta(hours=2)) sasurl = baseblobservice.make_blob_url('images', name, sas_token=sasToken) return sasurl
def _create_read_sas(cls, client, share=None, container=None): if (share and container) or (not share and not container): raise ValueError('set either share or container') if share: return client.generate_share_shared_access_signature( share, FilePermissions(read=True), datetime.utcnow() + timedelta(minutes=15)) elif container: return client.generate_container_shared_access_signature( container, BlobPermissions(read=True), datetime.utcnow() + timedelta(minutes=15))
def blob_service_generate_blob_shared_access_signature(account_name, storage_key, container_name, blob_name): blob_permissions = BlobPermissions(read=True) start = datetime.now() expiry = datetime.now() + timedelta(days=365) block_blob_service = get_block_blob_service(account_name, storage_key) return block_blob_service.generate_blob_shared_access_signature( container_name, blob_name, permission=blob_permissions, expiry=expiry, start=start, protocol='https')
def refresh_test(job_id: str): import requests storage = get_blob_storage_client() test_run = DbTestRun.query.filter_by(id=job_id).first() if not test_run: return "Test run job not found", 404 test_run_job = get_job(job_id) test_run.state = test_run_job.state.value if test_run_job.state == JobState.completed: for task in list_tasks(job_id): if task.id == 'test-creator': continue test_case = DbTestCase.query.filter_by( id=DbTestCase.get_full_name(task, test_run)).one_or_none() if test_case: continue test_case = DbTestCase(task, test_run) if not test_case.passed: # only load output of failed tests for performance reason container_name = 'output' blob_name = os.path.join(job_id, task.id, 'stdout.txt') sas = storage.generate_blob_shared_access_signature( container_name, blob_name, permission=BlobPermissions(read=True), protocol='https', expiry=(datetime.utcnow() + timedelta(hours=1))) url = storage.make_blob_url(container_name, blob_name, sas_token=sas, protocol='https') response = requests.request('GET', url) test_case.output = '\n'.join(response.text.split('\n')[58:-3]) db.session.add(test_case) db.session.commit() return redirect(url_for('test', job_id=job_id))
def api_hook(): import requests from morocco.core import get_batch_client, get_blob_storage_client if request.headers.get('X-Batch-Event') == 'test.finished': event = DbWebhookEvent(source='batch', content=request.data.decode('utf-8')) db.session.add(event) db.session.commit() job_id = request.form.get('job_id') task_id = request.form.get('task_id') test_run_job = get_job(job_id) test_run = DbTestRun.query.filter_by(id=job_id).first() test_run.state = test_run_job.state.value task = get_batch_client().task.get(job_id, task_id) test_case = DbTestCase(task, test_run) storage = get_blob_storage_client() if not test_case.passed: # only load output of failed tests for performance reason container_name = 'output' blob_name = os.path.join(job_id, task.id, 'stdout.txt') sas = storage.generate_blob_shared_access_signature( container_name, blob_name, permission=BlobPermissions(read=True), protocol='https', expiry=(datetime.utcnow() + timedelta(hours=1))) url = storage.make_blob_url(container_name, blob_name, sas_token=sas, protocol='https') response = requests.request('GET', url) test_case.output = '\n'.join(response.text.split('\n')[58:-3]) db.session.add(test_case) db.session.commit() return 'Update {} {}'.format(job_id, task_id), 200 return 'Unknown event', 400
def test_batch_copy_blob_with_sas(self): # create sas token for read permission on the source blob container sas_token = self._blob_service.generate_container_shared_access_signature( self._test_source_container, BlobPermissions(read=True), datetime.utcnow() + timedelta(minutes=15)) cmd_template = 'storage file copy start-batch' \ ' --source-container {}' \ ' --destination-share {}' \ ' --account-name {}' \ ' --account-key {}' \ ' --source-sas {}' _cli_main(cmd_template.format(self._test_source_container, self._test_target_share, self._file_service.account_name, self._file_service.account_key, sas_token)) files = list(glob_files_remotely(self._file_service, self._test_target_share, pattern=None)) assert len(files) == 41
def sync_build(commit: dict = None, sha: str = None, create_job=False): from datetime import datetime, timedelta from azure.batch.models import BatchErrorException, JobState from azure.storage.blob.models import BlobPermissions from morocco.core.services import (get_source_control_commit, get_source_control_commits, get_batch_client, get_blob_storage_client) from morocco.main import DbBuild, db from morocco.batch import create_build_job if not dict and not sha: raise ValueError('Missing commit') if not commit: if sha == '<latest>': commit = get_source_control_commits()[0] else: commit = commit or get_source_control_commit(sha) sha = commit['sha'] build_record = DbBuild.query.filter_by(id=sha).one_or_none() if build_record: build_record.update_commit(commit) else: build_record = DbBuild(commit=commit) db.session.add(build_record) try: batch_client = get_batch_client() batch_job = batch_client.job.get(sha) if create_job and batch_job.state == JobState.completed: batch_client.job.delete(sha) batch_job = create_build_job(sha) except BatchErrorException: if create_job: batch_job = create_build_job(sha) else: batch_job = None if batch_job: # build job can be deleted. it is not required to keep data in sync build_task = get_batch_client().task.get(job_id=sha, task_id='build') if not build_task: return 'Cloud task for the build is not found', 400 build_record.state = build_task.state.value storage = get_blob_storage_client() blob = 'azure-cli-{}.tar'.format(sha) if storage.exists(container_name='builds', blob_name=blob): build_record.build_download_url = storage.make_blob_url( 'builds', blob_name=blob, protocol='https', sas_token=storage.generate_blob_shared_access_signature( 'builds', blob, BlobPermissions(read=True), expiry=datetime.utcnow() + timedelta(days=365))) db.session.commit() return build_record
account_key= 'b/qWJCuFxdUD4A9Y6erFvXwqMcUBNJz+MAHHADXWN4v+8JRMxMfIW+nqeGKfUFhP1xcb5GJzA2OSuVEs3rVr0Q==' ) block_blob_service.create_blob_from_path( 'addresses', 'zoom_0.mp4', 'zoom_0.mp4', content_settings=ContentSettings(content_type='video/mp4')) #get url # block_blob_service.set_container_acl("addresses",{"AccessPolicy": "abc"}) sas_token = block_blob_service.generate_blob_shared_access_signature( "addresses", "zoom_0.mp4", permission=BlobPermissions().READ, expiry='2020-10-12', start=None) url = block_blob_service.make_blob_url( "addresses", "zoom_0.mp4", sas_token=sas_token, ) conn = http.client.HTTPSConnection("videobreakdown.azure-api.net") headers = { 'ocp-apim-subscription-key': "d7739fcadadc4280a02cbd6482f5ef86", 'content-type': "multipart/form-data", 'cache-control': "no-cache",