def update(self, area, selector, content_type, buffer): assert area is not None, 'area is none; should already be validated' area_config = config.load_area(area) storage_config = config.load_storage(area_config['storage']) area = area.lower() # httplib.HTTPConnection.debuglevel = 1 # http.client.HTTPConnection.debuglevel = 1 blob_service = BlockBlobService(account_name=storage_config['name'], account_key=storage_config['key1']) hash = base64.b64encode(hashlib.md5(buffer).digest()) content_settings = ContentSettings(content_md5=hash) if content_type is not None and len(content_type) > 0: content_settings.content_type = content_type blob_service.create_blob_from_bytes( area_config['container'], selector, buffer, content_settings=content_settings, validate_content=False ) return hash
def _upload_index_file(service, blob_name, title, links): print('Uploading index file {}'.format(blob_name)) service.create_blob_from_text( container_name=BLOB_CONTAINER_NAME, blob_name=blob_name, text= "<html><head><title>{0}</title></head><body><h1>{0}</h1>{1}</body></html>" .format( title, '\n'.join([ '<a href="{0}">{0}</a><br/>'.format(link) for link in links ])), content_settings=ContentSettings(content_type='text/html', content_disposition=None, content_encoding=None, content_language=None, content_md5=None, cache_control=None))
def append_blob( container: ContainerClient, blob_name: str, content_type: str, content_encoding: str, data: Any, return_sas_token: bool = True, metadata: Dict[str, str] = None, ) -> str: """ Uploads the given data to a blob record. If a blob with the given name already exist, it throws an error. Returns a uri with a SAS token to access the newly created blob. """ create_container_using_client(container) logger.info( f"Appending data to blob '{blob_name}'" + f"in container '{container.container_name}'" + f"on account: '{container.account_name}'" ) content_settings = ContentSettings( content_type=content_type, content_encoding=content_encoding ) blob = container.get_blob_client(blob_name) try: props = blob.get_blob_properties() if props.blob_type != BlobType.AppendBlob: raise Exception("blob must be an append blob") except exceptions.ResourceNotFoundError: props = blob.create_append_blob( content_settings=content_settings, metadata=metadata ) blob.append_block(data, len(data)) logger.debug(f" - blob '{blob_name}' appended. generating sas token.") if return_sas_token: uri = get_blob_uri_with_sas_token(blob) else: uri = remove_sas_token(blob.url) logger.debug(f" - blob access url: '{uri}'.") return uri
def __upload_file(self, container_name, filename, local_file, delete_local_file=False): blob_client = self.__blob_service_client.get_blob_client( container=container_name, blob=filename) with open(local_file, "rb") as data: blob = blob_client.upload_blob( data, content_settings=ContentSettings( content_type=self.__get_mime_type(local_file))) if delete_local_file: os.remove(local_file) return blob
def main(destblob: func.InputStream): logging.info( f"SetDestinationContentType blob trigger function processed blob \n" f"Name: {destblob.name}\n" f"Blob Size: {destblob.length} bytes") blob_name = os.path.basename(destblob.name) default_blob_byte_type = "application/octet-stream" source_connection_string = os.environ["source_connection_string"] source_container_name = os.environ["source_container_name"] destination_connection_string = os.environ["destination_connection_string"] destination_container_name = os.environ["destination_container_name"] destination_client = BlobClient.from_connection_string( destination_connection_string, destination_container_name, blob_name) current_content_type = destination_client.get_blob_properties( ).content_settings.content_type if (current_content_type == default_blob_byte_type): logging.info( f"Blob has content-type '{default_blob_byte_type}'. Sourcing the content-type from {source_container_name}..." ) source_client = BlobClient.from_connection_string( source_connection_string, source_container_name, blob_name) source_properties = source_client.get_blob_properties() source_content_type = source_properties.content_settings.content_type logging.info( f"Setting content-type for {blob_name} in {destination_container_name} to {source_content_type}..." ) destination_client.set_http_headers(content_settings=ContentSettings( content_type=source_content_type)) logging.info( f"Complete. Set content-type for {blob_name} in {destination_container_name}." ) else: logging.info( f"Blob already has a content-type '{current_content_type}'. Exiting." )
def stream_write(self, path, fp, content_type=None, content_encoding=None): blob_name = self._blob_name_from_path(path) content_settings = ContentSettings( content_type=content_type, content_encoding=content_encoding, ) try: self._blob_service.create_blob_from_stream( self._azure_container, blob_name, fp, content_settings=content_settings) except AzureException: logger.exception("Exception when trying to stream_write path %s", path) raise IOError("Exception when trying to stream_write path")
def _save(self, name, content): cleaned_name = clean_name(name) name = self._get_valid_path(name) params = self._get_content_settings_parameters(name, content) # Unwrap django file (wrapped by parent's save call) if isinstance(content, File): content = content.file content.seek(0) self.client.upload_blob(name, content, content_settings=ContentSettings(**params), max_concurrency=self.upload_max_conn, timeout=self.timeout, overwrite=self.overwrite_files) return cleaned_name
def upload_blob(storage_connection_string, container_name, blob_name, encoded_image): image = encoded_image.replace(' ', '+') # decoding base64 image from json body decoded_image = base64.b64decode(image) logging.debug(decoded_image) logging.debug('In upload blob function') #storage_account_name , storage_account_key = parse_conn_string(storage_connection_string) logging.debug('creating blob service client') blob_service_client = BlobServiceClient.from_connection_string( conn_str=storage_connection_string) # # create container if not present # if not container_name: # blob_service_client.create_container(container_name) # upload blob #BlobPermissions(container_name, blob_name, file_path, content_settings=None, # metadata=None, validate_content=False, progress_callback=None, max_connections=2, # lease_id=None, if_modified_since=None, if_unmodified_since=None, if_match=None, # if_none_match=None, timeout=None) #get container specific client logging.debug(f'getting client for container : {container_name}') container_client = blob_service_client.get_container_client( container=container_name) blob_client = container_client.get_blob_client(blob_name) if blob_client.exists(): blob_client.delete_blob() blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name) logging.debug('uploading blob now') try: blob_client.upload_blob(decoded_image) content_settings = ContentSettings(content_type='image/png') logging.debug(f'setting the content type : {content_settings}') # the below commented code works for standalone scripts not for functions # with open(full_path_to_file, "rb") as data: # blob_client.upload_data(data) # # blob_service_client.create_blob_from_path( # # container_name=container_name, # # blob_name=blob_name, # # file_path=full_path_to_file, # # ) except Exception as e: logging.error(str(e))
def put_item(self, fpath, relpath): identifier = generate_identifier(relpath) self._blobservice.create_blob_from_path( self.uuid, identifier, fpath, content_settings=ContentSettings(content_md5=_get_md5sum(fpath))) self._blobservice.set_blob_metadata(container_name=self.uuid, blob_name=identifier, metadata={ "relpath": relpath, "type": "item" }) return relpath
def test_create_blob_from_path_parallel_with_properties(self): # Arrange blob_name = self._get_blob_reference() data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings=ContentSettings( content_type='image/png', content_language='spanish') self.bs.create_blob_from_path(self.container_name, blob_name, FILE_PATH, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = self.bs.get_blob_properties(self.container_name, blob_name).properties self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
def uploadfile(block_blob_service, storage_container_name, filename, local_file_name, file_content_type): print("\n*****UPLOADING FILE*****") print(local_file_name) print('\n') #uploading file block_blob_service.retry = no_retry block_blob_service.create_blob_from_path( storage_container_name, filename, local_file_name, content_settings=ContentSettings(content_type=file_content_type), if_none_match='*', timeout='30') print("\n*****FILE UPLOADED*****")
def test_set_blob_properties_with_existing_blob(self): # Arrange blob_name = self._create_block_blob() # Act self.bs.set_blob_properties( self.container_name, blob_name, content_settings=ContentSettings(content_language='spanish', content_disposition='inline'), ) # Assert blob = self.bs.get_blob_properties(self.container_name, blob_name) self.assertEqual(blob.properties.content_settings.content_language, 'spanish') self.assertEqual(blob.properties.content_settings.content_disposition, 'inline')
def test_create_blob_from_bytes_with_index_and_count_and_properties(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') blob.upload_blob(data[3:], length=5, content_settings=content_settings) # Assert self.assertEqual(data[3:8], b"".join(list(blob.download_blob()))) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
def _save(self, name, content): cleaned_name = clean_name(name) name = self._get_valid_path(name) params = self._get_content_settings_parameters(name, content) # Unwrap django file (wrapped by parent's save call) if isinstance(content, File): content = content.file content.seek(0) self.service.create_blob_from_stream( container_name=self.azure_container, blob_name=name, stream=content, content_settings=ContentSettings(**params), max_connections=self.upload_max_conn, timeout=self.timeout) return cleaned_name
def _save(self, name, content): if hasattr(content.file, 'content_type'): content_type = content.file.content_type else: content_type = mimetypes.guess_type(name)[0] if hasattr(content, 'chunks'): content_data = b''.join(chunk for chunk in content.chunks()) else: content_data = content.read() content_settings = ContentSettings(content_type=content_type) self.connection.create_blob_from_bytes( self.azure_container, name, content_data, content_settings=content_settings) return name
def upload_file(STORAGE_NAME, STORAGE_KEY, NEW_CONTAINER_NAME, file, path, extension, content_type): """"create blob service, and upload files to container""" blob_service = BlockBlobService(account_name=STORAGE_NAME, account_key=STORAGE_KEY) try: blob_service.create_blob_from_path( NEW_CONTAINER_NAME, file, path, content_settings=ContentSettings(content_type=content_type + extension)) print("{} // BLOB upload status: successful".format(file)) except: print("{} // BLOB upload status: failed".format(file))
def write_on_blob_storage(data): data_bytes = data['base64Bytes'] conn_str = os.getenv('blob_storage_connection_string', None) container_name = os.getenv('blob_storage_container', None) blob_name = f"{uuid4()}.jpeg" if conn_str and container_name: # Create full Blob URL x = conn_str.split(';') image_url = f"{x[0].split('=')[1]}://{x[1].split('=')[1]}.{x[3].split('=')[1]}/{container_name}/{blob_name}" data['image_url'] = image_url # Upload data on Blob blob_client = BlobClient.from_connection_string( conn_str=conn_str, container_name=container_name, blob_name=blob_name) content_settings = ContentSettings(content_type='image/jpeg') blob_client.upload_blob(data_bytes, content_settings=content_settings) return data
def blob(): static_dir_path = "D:\home\site\wwwroot\static" account_name = 'hanastragetest' account_key = 'acount_key' container_name = 'images' container_url = "https://hanastragetest.blob.core.windows.net/" + container_name block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) app.logger.info("test message : {}".format(block_blob_service)) # container create block_blob_service.create_container(container_name) block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container) #app.logger.info("finish : block_blob_service.set_container_acl") files = os.listdir(static_dir_path) for file in files: # delete if block_blob_service.exists(container_name, file): block_blob_service.delete_blob(container_name, file) # blob write block_blob_service.create_blob_from_path( container_name, file, static_dir_path + '\\' + file, content_settings=ContentSettings(content_type='image/png')) # get container generator = block_blob_service.list_blobs(container_name) html = "" for blob in generator: #app.logger.info("generator : {}".format(blob.name)) html = "{}<img src='{}/{}'>".format(html, container_url, blob.name) #app.logger.info("generator_object : {}".format(generator)) result = { "result": True, "data": { "blob_name": [blob.name for blob in generator] } } return make_response(json.dumps(result, ensure_ascii=False) + html)
def init_blob_for_streaming_upload( container: ContainerClient, blob_name: str, content_type: str, content_encoding: str, data: Any, return_sas_token: bool = True, ) -> str: """ Uploads the given data to a blob record. If a blob with the given name already exist, it throws an error. Returns a uri with a SAS token to access the newly created blob. """ create_container_using_client(container) logger.info(f"Streaming blob '{blob_name}'" + f"to container '{container.container_name}' on account:" + f"'{container.account_name}'") content_settings = ContentSettings(content_type=content_type, content_encoding=content_encoding) blob = container.get_blob_client(blob_name) blob.stage_block() blob.commit_block_list() blob.upload_blob(data, content_settings=content_settings) logger.debug(f" - blob '{blob_name}' uploaded. generating sas token.") if return_sas_token: sas_token = generate_blob_sas( blob.account_name, blob.container_name, blob.blob_name, account_key=blob.credential.account_key, permission=BlobSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(days=14), ) uri = blob.url + "?" + sas_token else: uri = remove_sas_token(blob.url) logger.debug(f" - blob access url: '{uri}'.") return uri
def upload(globpath, container, queue, sas_token_env, storage_account_uri): try: sas_token_env = sas_token_env sas_token = os.getenv(sas_token_env) if sas_token is None: getLogger().error( "Sas token environment variable {} was not defined.".format( sas_token_env)) return 1 files = glob(globpath, recursive=True) for infile in files: blob_name = get_unique_name(infile, os.getenv('HELIX_WORKITEM_ID')) getLogger().info("uploading {}".format(infile)) blob_client = BlobClient( account_url=storage_account_uri.format('blob'), container_name=container, blob_name=blob_name, credential=sas_token) with open(infile, "rb") as data: blob_client.upload_blob(data, blob_type="BlockBlob", content_settings=ContentSettings( content_type="application/json")) if queue is not None: queue_client = QueueClient( account_url=storage_account_uri.format('queue'), queue_name=queue, credential=sas_token, message_encode_policy=TextBase64EncodePolicy()) queue_client.send_message(blob_client.url) getLogger().info("upload complete") return 0 except Exception as ex: getLogger().error('{0}: {1}'.format(type(ex), str(ex))) getLogger().error(format_exc()) return 1
def upload_whl(python_wheel_path, account_name, account_key, container_name): block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) blob_name = os.path.basename(python_wheel_path) block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path) nightly_build, local_version = parse_nightly_and_local_version_from_whl_name( blob_name) if local_version: html_blob_name = 'onnxruntime_{}_{}.html'.format( nightly_build, local_version) else: html_blob_name = 'onnxruntime_{}.html'.format(nightly_build) download_path_to_html = "./onnxruntime_{}.html".format(nightly_build) block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html) blob_name_plus_replaced = blob_name.replace('+', '%2B') with open(download_path_to_html) as f: lines = f.read().splitlines() new_line = '<a href="{blobname}">{blobname}</a><br>'.format( blobname=blob_name_plus_replaced) if new_line not in lines: lines.append(new_line) lines.sort() with open(download_path_to_html, 'w') as f: for item in lines: f.write("%s\n" % item) else: warnings.warn( "'{}' exists in {}. The html file is not updated.".format( new_line, download_path_to_html)) content_settings = ContentSettings(content_type='text/html') block_blob_service.create_blob_from_path(container_name, html_blob_name, download_path_to_html, content_settings=content_settings)
def uploadImg(myblob): from azure.storage.blob import BlockBlobService block_blob_service = BlockBlobService( account_name='eventdetect', account_key= 'VdqUAaFmd5K8bF5Pp+wt6cDfYUWiAtR2ib7+rKP76sqgJwSo0+friYmuVd+Y5oEWDh6/4oaRa423fXproar3aw==' ) block_blob_service.create_container('mycontainer') from azure.storage.blob import PublicAccess block_blob_service.create_container('mycontainer', public_access=PublicAccess.Container) block_blob_service.set_container_acl('mycontainer', public_access=PublicAccess.Container) from azure.storage.blob import ContentSettings block_blob_service.create_blob_from_path( 'mycontainer', myblob, 'Images\\' + myblob, content_settings=ContentSettings(content_type='image/jpg'))
def azureUpload(self, photoURL, instagramURL, photoName, blobName, containerClient, image, blobCreator): # Create a blob client using the local file name as the name for the blob blobClient = blobCreator.get_blob_client(container=containerClient, blob=image) # Upload the created file with open(image, "rb") as data: blobClient.upload_blob( data, content_settings=ContentSettings(content_type='jpg')) content = requests.get(loginJsonFile) data = json.loads(content.content) # Replace the site with cdn endpoint cdnLink = data["Login"]["cdnURL"] + blobName + "/" + image azureImageUrl = getImgUrlWithBlobSasToken(blobName, containerClient) # Go make the json with the new data, and slide in the cdn jsonCreator.JSONData(self, blobName, photoURL, instagramURL, photoName, cdnLink, azureImageUrl)
def create_blob(self): container_name = self._create_container() # Basic # Create a blob with no data blob_name1 = self._get_blob_reference() self.service.create_blob(container_name, blob_name1) # Properties settings = ContentSettings(content_type='html', content_language='fr') blob_name2 = self._get_blob_reference() self.service.create_blob(container_name, blob_name2, content_settings=settings) # Metadata metadata = {'val1': 'foo', 'val2': 'blah'} blob_name2 = self._get_blob_reference() self.service.create_blob(container_name, blob_name2, metadata=metadata) self.service.delete_container(container_name)
def upload(): f = request.files['file'] tit = request.form['title'] file_name = f.filename img_name = file_name.split('.') f.save('/Users/charith_gunuganti/Desktop/abcd' + file_name) location = '/Users/charith_gunuganti/Desktop/abcd' + file_name block_blob_service.create_blob_from_path('mycontainer', img_name[0], location, content_settings=ContentSettings(content_type='image/jpg')) cursor = conn.cursor() add_img = ("INSERT INTO imgstore " "(imgname, title, datetime, likes) " "VALUES (%(imgname)s, %(title)s, %(datetime)s, %(likes)s)") data_img = { 'imgname': img_name[0], 'title': tit, 'datetime': time.strftime('%Y-%m-%d %H:%M:%S'), 'likes': 0, }
def test_create_blob_from_bytes_with_properties(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act content_settings=ContentSettings( content_type='image/png', content_language='spanish') blob.upload_blob(data, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
def upload(request): if request.method == 'POST': form = UploadImageForm(request.POST, request.FILES) if form.is_valid(): image_filename = str(uuid.uuid4()) owner = None if request.user.is_authenticated: owner = request.user # Save image in Azure Blob storage img = Image.open(request.FILES['file']) #print("Image dimensions are", img.size) output_blob = BytesIO() img.convert('RGB').save(output_blob, format='JPEG') output_blob.seek(0) block_blob_service = BlockBlobService(account_name='magnifaistorage', account_key=os.getenv('AZ_STORAGE_KEY')) upload_container_name = 'uploads' if form.cleaned_data['should_colorize']: upload_container_name = 'uploads-colorize' block_blob_service.create_blob_from_stream(upload_container_name, image_filename + '.jpg', output_blob, content_settings=ContentSettings(content_type='image/jpeg')) #print("!!! Uploaded image to blob service.") # Create metadata for image in database metadata = ImageMetadata(filename=image_filename, should_colorize=form.cleaned_data['should_colorize'], owner=owner) #print("!!! Should colorize:", form.cleaned_data['should_colorize']) # Create shareable link for image in database link = ShareLink(name=str(uuid.uuid4()), image=metadata, owner=owner) metadata.save() #print("!!! Stored image metadata in database.") link.save() #print("!!! Stored shareable link data in database.") if request.user.is_authenticated: return redirect('modernaize:image', image_filename) else: return redirect('modernaize:share', link.name) #form.save() #return HttpResponseRedirect(reverse('modernaize:recent')) #return HttpResponse("Image uploaded successfully!") else: form = UploadImageForm() return render(request, 'modernaize/upload.html', { 'form': form, 'active_page': 'upload' })
def main(): args = parsing_options() az_conf = read_az_conf(args.azureConf) block_blob_service = BlockBlobService( account_name=az_conf['storage_account_name'], account_key=az_conf['storage_account_key']) block_blob_service.create_container('mycontainer') block_blob_service.create_blob_from_path( 'mycontainer', 'test.zip', '/tmp/test.zip', content_settings=ContentSettings( content_type='application/zip', content_md5='M2E5OTI1N2ZmMTRiNDExNzk1ZmFiMDEyZjQ3OGQ3ODIKi')) generator = block_blob_service.list_blobs('mycontainer') for blob in generator: print blob.name print blob.properties print blob.metadata
def upload_from_string(self, blob_path, string_data, content_type, bucket_type = "web"): """ Uploads the given string to azure blob storage service. :param blob_path: :param string_data: :param content_type: :param bucket_type: :return: """ if bucket_type == 'web': blob_client = self.azure_service_client.get_blob_client(container = self.azure_container_name, blob = blob_path) elif bucket_type == 'ml': blob_client = self.azure_service_client.get_blob_client(container = self.azure_container_name_ml, blob = blob_path) else: raise Exception('Invalid bucket_type, must be either web or ml.') my_content_settings = ContentSettings(content_type = content_type) blob_client.upload_blob(string_data, content_settings = my_content_settings)
def upload_blob(filename, url, account_key, inlineFlag=False): bc = BlobClient.from_blob_url(url, account_key) try: bc.get_blob_properties() print "Blob {} already exists, deleting".format(url) bc.delete_blob() except: None start = time.time() with open(filename, "rb") as data: # flags = {'Content-Disposition' : 'attachment '} # if inlineFlag: # flags['Content-Disposition'] = 'inline' settings = ContentSettings(content_type="image/jpeg", content_disposition="inline") bc.upload_blob(data, blob_type="BlockBlob", content_settings=settings) print "Uploaded {} in {} seconds".format(filename, time.time()-start)
def validate_azure_connection_read_write(self, bucket_name): test_file_path = 'diffgram_test_file.txt' log = regular_log.default() try: blob_client = self.connection_client.get_blob_client(container = bucket_name, blob = test_file_path) my_content_settings = ContentSettings(content_type = 'text/plain') blob_client.upload_blob('This is a diffgram test file', content_settings = my_content_settings, overwrite=True) except Exception as e: log['error']['azure_write_perms'] = 'Error Connecting to Azure: Please check you have write permissions on the Azure container.' log['error']['details'] = traceback.format_exc() return False, log try: shared_access_signature = BlobSharedAccessSignature( account_name = self.connection_client.account_name, account_key = self.connection_client.credential.account_key ) expiration_offset = 40368000 added_seconds = datetime.timedelta(0, expiration_offset) expiry_time = datetime.datetime.utcnow() + added_seconds filename = test_file_path.split("/")[-1] sas = shared_access_signature.generate_blob( container_name = bucket_name, blob_name = test_file_path, start = datetime.datetime.utcnow(), expiry = expiry_time, permission = BlobSasPermissions(read = True), content_disposition = 'attachment; filename=' + filename, ) sas_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format( self.connection_client.account_name, bucket_name, test_file_path, sas ) resp = requests.get(sas_url) if resp.status_code != 200: raise Exception( 'Error when accessing presigned URL: Status({}). Error: {}'.format(resp.status_code, resp.text)) except: log['error']['azure_write_perms'] = 'Error Connecting to Azure: Please check you have read permissions on the Azure container.' log['error']['details'] = traceback.format_exc() return False, log return True, log