class AzureStorage: def __init__(self, connectionString, container): self.BlobService = BlockBlobService(connection_string=connectionString) nameValue = UtilityHelper.connectStringToDictionary(connectionString) self.AccountName = nameValue['AccountName'] self.container = container def getBaseURL(self): return 'https://' + self.AccountName + '.blob.core.windows.net/' def uploadByLocalFile(self, localFullFileName, remoteBlobName): self.BlobService.create_blob_from_path(self.container, remoteBlobName, localFullFileName) blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName return blobURL def uploadByStream(self, streamData, remoteBlobName): self.BlobService.create_blob_from_stream(self.container, remoteBlobName, streamData) blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName return blobURL def uploadByBytes(self, bytesData, remoteBlobName): self.BlobService.create_blob_from_bytes(self.container, remoteBlobName, bytesData) blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName return blobURL def delete(self, blobName): self.BlobService.delete_blob(self.container, blobName) def copy(self, sourceBlobURL, targetBlobName): self.BlobService.copy_blob(self.container, targetBlobName, sourceBlobURL)
class BlobService(object): def __init__(self, blobonedge_module_name, blob_account_name, blob_account_key, image_container_name, image_upload_duration_sec, edge_id): localblob_connectionstring = 'DefaultEndpointsProtocol=http;BlobEndpoint=http://' + blobonedge_module_name + ':11002/' + blob_account_name + ';AccountName=' + blob_account_name + ';AccountKey=' + blob_account_key + ';' print("Try to connect to blob on edge by " + localblob_connectionstring) self.blockBlobService = BlockBlobService( endpoint_suffix='', connection_string=localblob_connectionstring) print("Connected to blob on edge") self.blockBlobService.create_container(image_container_name) print('Created image container - ' + image_container_name) self.imageContainerName = image_container_name self.imageUploadDurationSec = image_upload_duration_sec self.edgeId = edge_id self.blobLastUploadTime = time.time() def upload_image_to_blob(self, image): now = datetime.datetime.now() currentTime = time.time() print('Last Time:' + str(self.blobLastUploadTime) + '->Current:' + str(currentTime)) if (currentTime - self.blobLastUploadTime > self.imageUploadDurationSec): image_file_name = self.edgeId + "-img{0:%Y%m%d%H%M%S}".format( now) + ".jpg" # image_file_name = "image{0:%Y%m%d%H%M%S}".format(now) +".jpg" print('Uploading image as ' + image_file_name + ' at ' + str(currentTime)) self.blockBlobService.create_blob_from_stream( self.imageContainerName, image_file_name, image) print('Upload done') self.blobLastUploadTime = currentTime
def upload_file(): if request.method == 'POST': # check if the post request has the file part #file = request.files['file'] uploaded_files = request.files.getlist("file[]") filenames = [] for file in uploaded_files: if file.filename == '': flash('No file selected for uploading') return redirect(request.url) if file and allowed_file(file.filename): filename = secure_filename(file.filename) blobservice = BlockBlobService( account_name='flaskstorage', account_key= 'M9Hax/c6wKCdVXIcmBafad35/ctWW2OQJQynRMrM29D+mfZXWW53MF0Sthsf0cmWN+/XukVg/aZQ/6XBAB4cgg==' ) df = pd.read_excel(file.stream) data = BytesIO() df.to_excel(data, index=False) data = bytes(data.getvalue()) data = BytesIO(data) blobservice.create_blob_from_stream('htflaskcontainer', filename, data) data.close() filenames.append(filename) flash('File(s) successfully uploaded to Blob', 'filename') return redirect('/upload')
def upload_file(): if request.method == 'POST': # check if the post request has the file part #file = request.files['file'] uploaded_files = request.files.getlist("file[]") filenames = [] for file in uploaded_files: if file.filename == '': flash('No file selected for uploading') return redirect(request.url) if file and allowed_file(file.filename): filename = secure_filename(file.filename) blobservice = BlockBlobService( account_name='flaskstorage', account_key= '4+JwE+i1NvLF/oJuqmEVb0nNEiX0+9Tnq8M6U28cA0hsjP4qlpAfaSORdOg0Kphw2CWf/Zp4uPZG+M/sfdZytQ==' ) df = pd.read_excel(file.stream) data = BytesIO() df.to_excel(data, index=False) data = bytes(data.getvalue()) data = BytesIO(data) blobservice.create_blob_from_stream('htflaskcontainer', filename, data) data.close() filenames.append(filename) flash('File(s) successfully uploaded to Blob', 'filename') return redirect('/upload')
def pfile(answer, userId): f = urllib.request.urlopen( 'https://blobgroup1.blob.core.windows.net/datacontainer/User_data.json' ) dic = json.load(f) dic['UserInfo'][userId] = answer #f.write(json.dumps(dic)) ## Blob 接続準備 #account_nameにリソース名、account_keyに自分のアクセスキーを入力する block_blob_service = BlockBlobService( account_name='blobgroup1', account_key= 'KcyFmNHRvBwAEtaQ89C6vYFnRN1ftuTGiPcDjV1IOKBxmRmyxfb1ID97ucUNBbkD1BKqB0FgFrmAyC0d/eeThA==' ) # コンテナを作成する container_name = "datacontainer" block_blob_service.create_container(container_name) # コンテナのアクセス権限をpublicに設定する block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container) ## ファイルをblobに上げる #ファイルストリームを用いてファイルにバイナリデータを書き込む text = json.dumps(dic) file_data = io.BytesIO(text.encode("UTF-8")) #ファイルに名前をつける file_name = 'User_data.json' #ファイルの書き始め位置を指定(seek(0)は先頭から書き始めるという意味) file_data.seek(0) #blobを作成する block_blob_service.create_blob_from_stream(container_name, file_name, file_data) f.close()
class AzureBlobFeedStorage(BlockingFeedStorage): def __init__(self, uri): ''' Azure uses slashes '/' in their keys, which confuses the shit out of urlparse. So, we just handle it ourselves here. assuming format looks like this: azure://account_name:password@container/filename.jsonl azure://bobsaccount:1234567890abc1KUj0lK1gXHv4NHrCfKxfxHy3bwQJ+LqFHCay6r1S/Yhw2Ot4Tk6p1zF9IiMcPBo7o9poXZgA==@sites/filename.jsonl ''' container = uri.split('@')[1].split('/')[0] filename = '/'.join(uri.split('@')[1].split('/')[1::]) account_name, account_key = uri[8::].split('@')[0].split(':') self.account_name = account_name self.account_key = account_key self.container = container self.filename = filename self.blob_service = BlockBlobService(account_name=self.account_name, account_key=self.account_key) def _store_in_thread(self, file): file.seek(0) self.blob_service.create_blob_from_stream(self.container, self.filename, file)
def transfer_fileshare_to_blob(config, fileshare_uri, output_model_name): ''' NB -- transfer proceeds via local temporary file! ''' file_service = FileService(config.storage_account_name, config.storage_account_key) blob_service = BlockBlobService(config.storage_account_name, config.storage_account_key) blob_service.create_container(config.container_trained_models) blob_service.create_container(config.predictions_container) uri_core = fileshare_uri.split('.file.core.windows.net/')[1].split('?')[0] fields = uri_core.split('/') fileshare = fields.pop(0) subdirectory = '/'.join(fields[:-1]) file_name = '{}/{}'.format(output_model_name, fields[-1]) with TemporaryFile() as f: file_service.get_file_to_stream(share_name=fileshare, directory_name=subdirectory, file_name=fields[-1], stream=f) f.seek(0) if 'predictions' in fields[-1]: blob_service.create_blob_from_stream( config.predictions_container, '{}_predictions_test_set.csv'.format(output_model_name), f) else: blob_service.create_blob_from_stream( config.container_trained_models, file_name, f) return
def uploadFile(): accountey = "xxxx" accountName = "xxxxx" containerName = "xxx-xxx" blobService = BlockBlobService(account_name=accountName, account_key=accountey) blobService.create_container(containerName) blobService.create_blob_from_stream(containerName, "image4.jpg", file.raw)
def _save_to_azure(dest_path, file_obj, container): blob_service = BlockBlobService(connection_string=settings.AZURE_STORAGE_CONNECTION_STRING) blob_service.create_blob_from_stream(container, dest_path, file_obj) sas_token = blob_service.generate_blob_shared_access_signature( container, dest_path, BlobPermissions.READ,datetime.utcnow() + timedelta(hours=24*3000)) blob_url = blob_service.make_blob_url(container, dest_path, sas_token=sas_token) return blob_url, blob_url
def write_blob(self, sas_uri, blob_name, input_stream): sas_service = BlockBlobService( account_name=self.get_account_from_uri(sas_uri), sas_token=self.get_sas_key_from_uri(sas_uri)) container_name = self.get_container_from_uri(sas_uri) sas_service.create_blob_from_stream(container_name, blob_name, input_stream) return sas_service.make_blob_url(container_name, blob_name, sas_token=self.get_sas_key_from_uri(sas_uri))
def push_dataframe_to_azure_blob(df, account_name, account_key, container_name, file_name): data = StringIO() df.to_csv(data) data = bytes(data.getvalue(), 'utf-8') data = BytesIO(data) blob_service = BlockBlobService(account_name=account_name, account_key=account_key) blob_service.create_blob_from_stream(container_name=container_name, blob_name=file_name, stream=data) data.close()
class AzureStore(DataStoreInterface): service_cache = _ClientCache() def __init__(self): service = self.service_cache.get('azure') if service is None: self.service = BlockBlobService(**env.AZURE_STORAGE_CREDENTIALS) else: self.service: BlockBlobService = service self.service_cache.get('azure') self.service_cache.put('azure', self.service) def get(self, sample_url: str) -> BinaryIO: container_name, blob_name = self._parser_url(sample_url) stream = io.BytesIO() self.service.get_blob_to_stream(container_name, blob_name, stream) stream.seek(0) return stream def put(self, sample_url: str, stream: BinaryIO): container_name, blob_name = self._parser_url(sample_url) self.service.create_container(container_name) self.service.create_blob_from_stream(container_name, blob_name, stream) def exists(self, sample_url) -> bool: container_name, blob_name = self._parser_url(sample_url) return self.service.exists(container_name, blob_name) def delete(self, sample_url): container_name, blob_name = self._parser_url(sample_url) self.service.delete_blob(container_name, blob_name) def get_signed_url(self, sample_url) -> str: container_name, blob_name = self._parser_url(sample_url) signature = self.service.generate_blob_shared_access_signature( container_name, blob_name, permission=BlobPermissions(read=True), expiry=datetime.datetime.utcnow() + datetime.timedelta(minutes=15), ) url = self.service.make_blob_url(container_name, blob_name, protocol='https', sas_token=signature) return url @staticmethod def _parser_url(url) -> Tuple[str, str]: if not url.startswith(AZURE_PREFIX): raise ValueError(f"URL should start with {AZURE_PREFIX}") container_name, blob_name = url[len(AZURE_PREFIX):].split('/', maxsplit=1) return container_name, blob_name
def send_data_to_blob_cloud(self, data, blob_name): from azure.storage.blob.baseblobservice import _get_path blob_service = BlockBlobService(account_name=ACCOUNT_NAME, sas_token=SAS_TOKEN) blob_service.create_blob_from_stream( CONTAINER_NAME, blob_name, data, content_settings=ContentSettings( content_type='application/zip', content_disposition='attachment; filename="nch-outfile.zip"')) return '{}://{}{}'.format(blob_service.protocol, blob_service._get_host(), _get_path(CONTAINER_NAME, blob_name))
class AzureStorageService(): def __init__(self): self.account_name = config.get( 'ckanext.dataextractor.azure_storage_account_name', None) self.account_key = config.get( 'ckanext.dataextractor.azure_storage_account_key', None) self.container_name = config.get( 'ckanext.dataextractor.azure_storage_container_name', None) self.blob_expiration_days = config.get( 'ckanext.dataextractor.blob_expiration_days', 10) if not self.account_name or not self.account_key or not self.container_name: raise ValueError( 'azure_storage_account_name, azure_storage_account_key, ' 'azure_storage_container_name') self.service = BlockBlobService(account_name=self.account_name, account_key=self.account_key) def _get_blob_reference(self, prefix, format): prefix = prefix.replace('/', '') return '{}{}.{}'.format(prefix + '-', str(uuid.uuid4()).replace('-', ''), format.lower()) def blob_create(self, stream, format, resource_name): blob_name = self._get_blob_reference(resource_name, format) self.service.create_blob_from_stream(self.container_name, blob_name, stream) blob_url = self.service.make_blob_url(self.container_name, blob_name) return blob_url def blobs_delete(self): blobs = self.service.list_blobs(self.container_name) blobs_deleted = 0 for blob in blobs: time_diff = datetime.now() - blob.properties.last_modified.replace( tzinfo=None) if time_diff.days >= int(self.blob_expiration_days): blobs_deleted += 1 self.service.delete_blob(self.container_name, blob.name) return blobs_deleted
class AzureStorage(BlobStorage): """Azure storage provider that utilizes the Azure blob storage. Args: connection_string: See http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ for the connection string format. container_name: the name of the blob container in which all blobs are stored. """ def __init__(self, connection_string, container_name): self._service = BlockBlobService(connection_string=connection_string) if not self._service.exists(container_name): raise ValueError("Container does not exist: " + container_name) self._container_name = container_name def get_object(self, blob_name): blob = self._service.get_blob_to_text(self._container_name, blob_name) return json.loads(blob.content) def put_object(self, obj, blob_name): data = json.dumps(obj).encode("utf-8") self._service.create_blob_from_bytes(self._container_name, blob_name, data) return Blob(blob_name, len(data)) @contextlib.contextmanager def get_file(self, blob_name): try: stream = AzureBlobReader(self._service, self._container_name, blob_name) yield stream finally: stream.close() def put_file(self, fileobj, blob_name): self._service.create_blob_from_stream(self._container_name, blob_name, fileobj) size = fileobj.tell() return Blob(blob_name, size) def put_avro(self, schema, records, blob_name, codec='snappy'): writer = AzureBlobWriter(self._service, self._container_name, blob_name) fastavro.writer(writer, schema, records, codec) writer.close() size = writer.tell() return Blob(blob_name, size)
def main(req: func.HttpRequest) -> func.HttpResponse: # 1. Get Header Values account_name = req.headers.get('account_name') account_key = req.headers.get('account_key') input_container = req.headers.get('input_container') output_container = req.headers.get('output_container') blob_name = req.headers.get('blob_name') if None not in [ account_name, account_key, input_container, output_container, blob_name ]: # 2. Get PDF from Azure Blob Storage block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) stream = io.BytesIO() block_blob_service.get_blob_to_stream(container_name=input_container, blob_name=blob_name, stream=stream) inputpdf = PdfFileReader(stream) # 3. Loop through each page in the PDF for i in range(inputpdf.numPages): output = PdfFileWriter() output.addPage(inputpdf.getPage(i)) output_stream = io.BytesIO() output.write(output_stream) output_stream.seek(0) output_blob_name = blob_name[:-4] + "%s.pdf" block_blob_service.create_blob_from_stream( container_name=output_container, blob_name=output_blob_name % i, stream=output_stream) # 4. Return HTTP Response body = { 'account_name': account_name, 'input_container': input_container, 'output_container': output_container, 'blob_name': blob_name, 'account_key': 'YOUR_SECRET_ACCOUNT_KEY' if account_key else None } return func.HttpResponse(json.dumps(body), headers={'Content-Type': 'application/json'})
def save_file_to_blob(file_name): from azure.storage.blob import BlockBlobService, PublicAccess container_name = 'stockrawdata' storage_name = 'stockanalysisadls' # try: block_blob_service = BlockBlobService( account_name=storage_name, account_key= 'mDmjRCgeoaxYtLf69kXRUVj6PO9gZq4e/1/LHyU4aqMFaOC1AusTRCH4l5ApJDSjNzA1fQAkDviuSLIkII1a2g==' ) block_blob_service.create_blob_from_stream(container_name, 'raw/nasdaq/' + file_name, get_nasdaq_data(file_name)) except Exception as err: print("Error occurred while loading data to ADLS. Error Details ::" + str(err))
def WriteBlob(self, blob_name, value): """ 単一 BLOB ファイルを作成しテキストを保存する。 """ szRet = "" if (LogWriter.DEBUG_MODE): return ("Debug モードのため書き込みをしません。") try: #blob_name = r'sample.txt'; szRet = "BlockBlobService" blob_service = BlockBlobService(self._name, self._key) szRet = "create_container" blob_service.create_container(LogWriter.LOG_CONTAINER_NAME, public_access=PublicAccess.Blob) szRet = "create_blob_from_bytes" #blob_service.create_blob_from_bytes( # log_container_name, # log_blob_name, # b'<center><h1>Hello World!</h1></center>', # content_settings=ContentSettings('text/html') #) if (isinstance(value, str)): szRet = "create_blob_from_text" blob_service.create_blob_from_text( LogWriter.LOG_CONTAINER_NAME, blob_name, value) else: szRet = "create_blob_from_stream" blob_service.create_blob_from_stream( LogWriter.LOG_CONTAINER_NAME, blob_name, io.BytesIO(value)) #}if #szRet = "make_blob_url" #print(blob_service.make_blob_url(log_container_name, log_blob_name)) szRet = "OK" except: print(r"Exception.") #try return szRet
def upload(request): if request.method == 'POST': form = UploadImageForm(request.POST, request.FILES) if form.is_valid(): image_filename = str(uuid.uuid4()) owner = None if request.user.is_authenticated: owner = request.user # Save image in Azure Blob storage img = Image.open(request.FILES['file']) #print("Image dimensions are", img.size) output_blob = BytesIO() img.convert('RGB').save(output_blob, format='JPEG') output_blob.seek(0) block_blob_service = BlockBlobService(account_name='magnifaistorage', account_key=os.getenv('AZ_STORAGE_KEY')) upload_container_name = 'uploads' if form.cleaned_data['should_colorize']: upload_container_name = 'uploads-colorize' block_blob_service.create_blob_from_stream(upload_container_name, image_filename + '.jpg', output_blob, content_settings=ContentSettings(content_type='image/jpeg')) #print("!!! Uploaded image to blob service.") # Create metadata for image in database metadata = ImageMetadata(filename=image_filename, should_colorize=form.cleaned_data['should_colorize'], owner=owner) #print("!!! Should colorize:", form.cleaned_data['should_colorize']) # Create shareable link for image in database link = ShareLink(name=str(uuid.uuid4()), image=metadata, owner=owner) metadata.save() #print("!!! Stored image metadata in database.") link.save() #print("!!! Stored shareable link data in database.") if request.user.is_authenticated: return redirect('modernaize:image', image_filename) else: return redirect('modernaize:share', link.name) #form.save() #return HttpResponseRedirect(reverse('modernaize:recent')) #return HttpResponse("Image uploaded successfully!") else: form = UploadImageForm() return render(request, 'modernaize/upload.html', { 'form': form, 'active_page': 'upload' })
class AzureBlobFeedStorage(BlockingFeedStorage): def __init__(self, uri): from azure.storage.blob import BlockBlobService container = uri.split("@")[1].split("/")[0] filename = "/".join(uri.split("@")[1].split("/")[1::]) account_name, account_key = uri[8::].split("@")[0].split(":") self.account_name = account_name self.account_key = account_key self.container = container self.filename = filename self.blob_service = BlockBlobService(account_name=self.account_name, account_key=self.account_key) def _store_in_thread(self, file): file.seek(0) self.blob_service.create_blob_from_stream(self.container, self.filename, file)
class AzureBlobFeedStorage(BlockingFeedStorage): """ Pulled from https://github.com/curabase/scrapy-feedexporter-azure-blob/ """ def __init__(self, uri): container = uri.split('@')[1].split('/')[0] filename = '/'.join(uri.split('@')[1].split('/')[1::]) account_name, account_key = uri[8::].split('@')[0].split(':') self.account_name = account_name self.account_key = account_key self.container = container self.filename = filename self.blob_service = BlockBlobService( account_name=self.account_name, account_key=self.account_key, ) def _store_in_thread(self, file): file.seek(0) self.blob_service.create_blob_from_stream(self.container, self.filename, file)
class BlobSaveable(Saveable): def __init__(self, account_name, account_key, container_name): self.container_name = container_name self.blob_service = BlockBlobService(account_name=account_name, account_key=account_key) def save(self, relpath, name, image): # save image to bytes stream im = Image.fromarray(image) stream = io.BytesIO() im.save(stream, format='png') stream.seek(0) # upload blob blob_name = '{}/{}.png'.format(relpath, name) self.blob_service.create_blob_from_stream(self.container_name, blob_name, stream) # return path path = '{}://{}/{}/{}'.format(self.blob_service.protocol, self.blob_service.primary_endpoint, self.container_name, blob_name) return path
def convert_file(url): image = Image.open(BytesIO(requests.get(url).content)).resize((IMAGE_WIDTH, IMAGE_HEIGHT)) image_array = np.array(image) augmentation = ElasticTransform() data = {"image": image_array} augmented = augmentation(**data) image = augmented["image"] blob_service = BlockBlobService(account_name=account, account_key=key) img = Image.fromarray(image) imagefile = BytesIO() img.save(imagefile, format='PNG') imagefile.seek(0) Randomfilename = id_generator() filename = Randomfilename + '.png' blob_service.create_blob_from_stream(container, filename, imagefile) ref = 'https://'+ account + '.blob.core.windows.net/' + container + '/' + filename redis_client.set(url, ref)
class DirectoryClient: def __init__(self, connection_string, container_name): self.container_name = container_name # service_client = BlobServiceClient.from_connection_string(connection_string) # self.client = service_client.get_container_client(container_name) self.client = BlockBlobService(account_name=account_name, account_key=account_key) # send big .zip file to blob just from filestream (no downloading) def create_blob_from_stream(self, blob_name, stream): print('in azure_blob from stream function') self.client.create_blob_from_stream(self.container_name, blob_name, stream) return def upload_file(self, source, dest): ''' Upload a single file to a path inside the container ''' # print(f'Uploading {source} to {dest}') # do i need this with open thing??? not too sure bc i never use the data var with open(source, 'rb') as data: self.client.create_blob_from_path(self.container_name, dest, source) return def download(self, source, dest): ''' Download a file or directory to a path on the local filesystem ''' if not dest: raise Exception('A destination must be provided') blobs = self.ls_files(source, recursive=True) print('blobs to download:', blobs) if blobs: # if source is a directory, dest must also be a directory if not source == '' and not source.endswith('/'): source += '/' if not dest.endswith('/'): dest += '/' # append the directory name from source to the destination dest += os.path.basename(os.path.normpath(source)) + '/' blobs = [source + blob for blob in blobs] for blob in blobs: blob_dest = dest + os.path.relpath(blob, source) # print("\nDownloading blob to " + blob_dest) self.client.get_blob_to_path(self.container_name, blob, blob_dest) else: self.client.get_blob_to_path(self.container_name, source, dest) def download_file(self, source, dest): ''' # Download a single file to a path on the local filesystem ''' # dest is a directory if ending with '/' or '.', otherwise it's a file if dest.endswith('.'): dest += '/' blob_dest = dest + os.path.basename(source) if dest.endswith( '/') else dest # print( f'Downloading {source} to {blob_dest}') os.makedirs(os.path.dirname(blob_dest), exist_ok=True) self.client.get_blob_to_path(self.container_name, source, blob_dest) '''data = bc.download_blob() file.write(data.readall())''' def ls_files(self, path, recursive=False): ''' List files under a path, optionally recursively ''' if not path == '' and not path.endswith('/'): path += '/' blob_iter = self.client.list_blobs(self.container_name, prefix=path) files = [] for blob in blob_iter: relative_path = os.path.relpath(blob.name, path) if recursive or not '/' in relative_path: files.append(relative_path) return files def ls_dirs(self, path, recursive=False): ''' List directories under a path, optionally recursively ''' if not path == '' and not path.endswith('/'): path += '/' blob_iter = self.client.list_blobs(self.container_name, prefix=path) dirs = [] for blob in blob_iter: relative_dir = os.path.dirname(os.path.relpath(blob.name, path)) if relative_dir and (recursive or not '/' in relative_dir) and not relative_dir in dirs: dirs.append(relative_dir) return dirs def rm(self, path, recursive=False): ''' Remove a single file, or remove a path recursively ''' if recursive: self.rmdir(path) else: print(f'Deleting {path}') self.client.delete_blob(path) def rmdir(self, path): ''' Remove a directory and its contents recursively ''' blobs = self.ls_files(path, recursive=True) if blobs == []: return if not path == '' and not path.endswith('/'): path += '/' blobs = [path + blob for blob in blobs] print(f'Deleting blobs in ' + str(self.container_name)) for blob in blobs: self.client.delete_blob(blob_name=blob, container_name=self.container_name)
ASGneg = pd.concat([df1, df2, df3, df4, df5, df10, df15, df20]) ASGINC = pd.concat([ASGpos, ASGneg]) ASGINC = ASGINC.rename(columns={'Unnamed: 0': 'Date'}) from io import StringIO, BytesIO from azure.storage.blob import BlockBlobService blobservice = BlockBlobService( account_name='flaskstorage', account_key= '4+JwE+i1NvLF/oJuqmEVb0nNEiX0+9Tnq8M6U28cA0hsjP4qlpAfaSORdOg0Kphw2CWf/Zp4uPZG+M/sfdZytQ==' ) data = BytesIO() ASGINC.to_excel(data, index=False) data = bytes(data.getvalue()) data = BytesIO(data) blobservice.create_blob_from_stream('mtscontainer', 'MTSINC.xlsx', data) data.close() """ from pandas import ExcelWriter if not os.path.exists('Generated-Excels'): os.makedirs('Generated-Excels') writer = ExcelWriter('Generated-Excels/MTSINCR.xlsx') ASGINC.to_excel(writer,'Sheet2') writer.save() print('done') """
def upload_res(path, local_path): BlockBlobService = connect_azure() Container_name = "open-pai" path = BlockBlobService.create_blob_from_stream(Container_name, path, local_path) print("Upload success!") return path
class AzureStorage(plugins.IStoragePlugin): def configure(self, config): self.storage = BlockBlobService(account_name=config['account_name'], account_key=config['account_key']) self.container = config['container'] try: container = self.storage.get_container_properties(self.container) log.info("Configuring Azure blob storage %s/%s", self.storage.account_name, self.container) except AzureMissingResourceHttpError as e: log.warning("Container '%s' is missing in account '%s', trying to create new", self.container, self.storage.account_name) try: self.storage.create_container(self.container) self.storage.set_container_acl(self.container, public_access=PublicAccess.Container) except Exception as e: log.critical("Cannot create new container: %s", e) raise plugins.PluginInitException("Cannot create new container") except AzureHttpError as e: log.critical("Cannot access container '%s' in account '%s': %s", self.container, self.storage.account_name, e) raise plugins.PluginInitException("Cannot access container") except Exception as e: log.critical("Cannot access container '%s' in account '%s': %s", self.container, self.storage.account_name, e) raise plugins.PluginInitException("Cannot access container") def delete(self, key): log.info("Deleting file '%s' from %s/%s", key, self.storage.account_name, self.container) try: self.storage.delete_blob(self.container, key) except AzureMissingResourceHttpError: log.error("File '%s' was not found in %s/%s", key, self.storage.account_name, self.container) raise common.NotFound('File not found') except Exception as e: log.error("Cannot delete '%s' from %s/%s: %s", key, self.storage.account_name, self.container, e) raise common.FatalError(e) def put(self, key, filename=None, file=None): storage_key = key try: if filename: log.debug("Uploading %s to %s", filename, self.storage.make_blob_url(self.container, storage_key)) self.storage.create_blob_from_path(self.container, storage_key, filename, content_settings=ContentSettings(content_type='application/octet-stream')) elif file: old_pos = file.tell() file.seek(0) log.debug("Uploading from stream to %s", self.storage.make_blob_url(self.container, storage_key)) self.storage.create_blob_from_stream(self.container, storage_key, file, content_settings=ContentSettings(content_type='application/octet-stream')) file.seek(old_pos) except Exception as e: # TODO: more detailed error inspection log.critical("Error uploading to %s/%s: %s", self.storage.account_name, self.container, e) raise common.FatalError(e) return storage_key def get(self, key, stream): # current azure python sdk barely can work with non-seekable streams, # so we have to implement chunking by our own # TODO: proper ranging? RFC says server SHOULD return 406 once range is unsatisfiable, # but Azure is OK with end pos > blob length unless blob is not empty chunk_size = 4*1024*1024 chunk_start = 0 chunk_end = chunk_size - 1 while True: try: chunk = self.storage._get_blob(self.container, key, start_range=chunk_start, end_range=chunk_end) log.debug("Writing %s bytes from %s", len(chunk.content), chunk_start) stream.write(chunk.content) except IOError: # remote side closed connection return except AzureMissingResourceHttpError as e: raise common.NotFound(e) except (AzureHttpError, AzureException) as e: raise common.TemporaryError('Error while downloading {}: {}'.format(key, e)) blob_length = int(chunk.properties.content_range.split('/')[1]) chunk_start, chunk_end, blob_size = map(int, re.match(r'^bytes\s+(\d+)-(\d+)/(\d+)$', chunk.properties.content_range).groups()) if chunk_end == blob_size - 1: # no more data to stream break else: chunk_start = chunk_end + 1 chunk_end += chunk_size return 0
print('It didnt work') ASGneg = pd.concat([df1, df2, df3, df4, df5, df10, df15, df20]) ASGINC = pd.concat([ASGpos, ASGneg]) ASGINC = ASGINC.rename(columns={'Unnamed: 0': 'Date'}) from io import StringIO, BytesIO from azure.storage.blob import BlockBlobService blobservice = BlockBlobService( account_name='flaskstorage', account_key= '4+JwE+i1NvLF/oJuqmEVb0nNEiX0+9Tnq8M6U28cA0hsjP4qlpAfaSORdOg0Kphw2CWf/Zp4uPZG+M/sfdZytQ==' ) data = BytesIO() ASGINC.to_excel(data, index=False) data = bytes(data.getvalue()) data = BytesIO(data) blobservice.create_blob_from_stream('mobilecontainer', 'MOBILEINC.xlsx', data) data.close() """ from pandas import ExcelWriter if not os.path.exists('Generated-Excels'): os.makedirs('Generated-Excels') writer = ExcelWriter('Generated-Excels/MOBILEINCR.xlsx') ASGINC.to_excel(writer,'Sheet2') writer.save() print('done') """
from PIL import Image import io import requests api_key="your_key" from apiclient.discovery import build import azure from azure.storage.blob import BlockBlobService from azure.storage.blob import PublicAccess from azure.storage.blob import ContentSettings block_blob_service =BlockBlobService(account_name="your_account_name",account_key="account_key") resource=build("customsearch","v1",developerKey=api_key).cse() images=[] for i in range(1,50,10): result=resource.list(q="your_query",cx="cx_of_your_site",searchType="image",start=i).execute() images+=result["items"] for item in images: response=requests.get(item["link"]) block_blob_service.create_blob_from_stream('account_name','external',io.BytesIO(response.content),content_settings=ContentSettings(content_type='image/Jpeg'))
ASGneg = pd.concat([df1, df2, df3, df4, df5, df10, df15, df20]) ASGINC = pd.concat([ASGpos, ASGneg]) ASGINC = ASGINC.rename(columns={'Unnamed: 0': 'Date'}) blobservice = BlockBlobService( account_name='flaskstorage', account_key= 'M9Hax/c6wKCdVXIcmBafad35/ctWW2OQJQynRMrM29D+mfZXWW53MF0Sthsf0cmWN+/XukVg/aZQ/6XBAB4cgg==' ) data = BytesIO() ASGINC.to_excel(data, index=False) data = bytes(data.getvalue()) data = BytesIO(data) blobservice.create_blob_from_stream('totalcontainer', 'TOTALINC.xlsx', data) data.close() """ from pandas import ExcelWriter if not os.path.exists('Generated-Excels'): os.makedirs('Generated-Excels') writer = ExcelWriter('Generated-Excels/TOTALINCR1.xlsx') ASGINC.to_excel(writer,'Sheet2') writer.save() print('done') """ "" #*********************************Attrition for total************************************************* revdf = ser revdf['ds'] = revdf.index revdf = revdf.rename(columns={"Attrition": 'y'}) my_model = Prophet(interval_width=0.95, changepoint_prior_scale=5)
from azure.storage.blob import BlockBlobService from azure.storage.blob import PublicAccess from azure.storage.blob import ContentSettings import requests import io AzureStorageAccount = "istagingstorage" AzureStorageAccessKey = "qkFU/ah2v4cHvQ7oAZASb2HRGFUkJhg2xs5KBYB+2fEnYmSp6hZH9U3vEO6TujzHHdBF3HWVgqalwcUuvIBMUQ==" AzureStorageContainerName = "peter-container" blobname="out2.jpg" url = "https://www.gravatar.com/avatar/dd071a6a7c97ba637c558c5e71137c7b?s=32&d=identicon&r=PG" block_blob_service = BlockBlobService(account_name=AzureStorageAccount, account_key=AzureStorageAccessKey) r = requests.get(url, stream=True) stream = io.BytesIO(r.content) block_blob_service.create_blob_from_stream(AzureStorageContainerName,blobname,stream) def _getDownloadlink(): baseurl = "https://"+AzureStorageAccount+".blob.core.windows.net/" downloadlink = baseurl+AzureStorageContainerName+"/"+blobname return downloadlink print _getDownloadlink()
print('\n*** Pushing selected file to Azure ***\n') print(' - path to input file: ' + input_filename) print(' - output filename: ' + output_filename) # Azure commands from azure.storage.blob import BlockBlobService from azure.storage.blob import PublicAccess azu_dict = Lfun.csv_to_dict(Ldir['data'] + 'accounts/azure_pm_2015.05.25.csv') account = azu_dict['account'] key = azu_dict['key'] blob_service = BlockBlobService(account_name=account, account_key=key) blob_service.create_container blob_service.set_container_acl(container_name, public_access=PublicAccess.Container) # write it to Azure try: bname = open(input_filename, 'rb') blob_service.create_blob_from_stream(container_name, output_filename, bname) bname.close() print('\n*** success ***') az_url = ('https://pm2.blob.core.windows.net/' + container_name + '/' + output_filename) print('\nUSE THIS URL TO ACCESS THE FILE\n') print(az_url) except: # could be FileNotFoundError from open, or an Azure error print('*** FAILED TO WRITE ***') print('\n' + 50*'*')
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') logging.info(f'Method: {req.method}') if req.method == "OPTIONS": return func.HttpResponse(status_code=204, headers={ 'Access-Control-Allow-Headers': 'content-type', 'Access-Control-Allow-Methods': 'POST', 'Access-Control-Max-Age': '180', 'Access-Control-Allow-Origin': '*' }) body = req.get_json() blob_service = BlockBlobService(account_name=storageAccount, account_key=storageAccountKey) # prep trainer trainer = CustomVisionTrainingClient(trainingKey, apiEndpoint) #check tags check_tags(trainer) records = {'images': []} image_list = [] try: for item in body['items']: # sign sign = item['type'].strip() # image bits img = base64.b64decode(item['image'].replace( 'data:image/png;base64,', '')) stream = BytesIO(img) # storage path + save image_name = f'{str(uuid.uuid4())}.png' blob_name = f'{base_folder}/{sign}/{image_name}' sresponse = blob_service.create_blob_from_stream( storageContainer, blob_name, stream) logging.info(f'Storage Response: {sresponse}') # save to custom vision image_list.append( ImageFileCreateEntry(name=image_name, contents=img, tag_ids=[tags[sign].id])) # return image path = f'{blob_service.protocol}://{blob_service.primary_endpoint}/{storageContainer}/{blob_name}' records['images'].append({'sign': sign, 'path': path}) # save list upload_result = trainer.create_images_from_files(projectId, images=image_list) if not upload_result.is_batch_successful: records['error'] = {'type': 'CustomVision Error', 'items': []} for image in upload_result.images: records['error']['items'].append( {image.source_url: image.status}) else: records['error'] = {} except Exception as error: logging.exception('Python Error') records['error'] = { 'code': '500', 'message': f'{type(error).__name__}: {str(error)}', 'type': 'Python Error' } return func.HttpResponse(body=json.dumps(records), status_code=200, headers={ 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' })