def save_image(request, filefield): blob_client = BlockBlobService( account_name=appvar.config["BLOB_ACCT_NAME"], account_key=appvar.config["BLOB_KEY"]) filestorage = request.files.get(filefield) if filestorage is None: raise FileNotFoundError("File storage") if filestorage.filename == "": raise FileNotFoundError("File blank") file_name, ext = os.path.splitext(filestorage.filename) file_name_rand = file_name + str(uuid.uuid4()) + ext file_name_secure = secure_filename(file_name_rand) content = filestorage.read() blob_client.create_blob_from_bytes(container_name="ocrimages", blob_name=file_name_secure, blob=content) return file_name_secure
class BlobUploader(object): """ A simple helper class for uploading image data to blob storage. """ def __init__(self, container_name, connection_string): """ Initialize a new BlobUploader. """ self.container_name = container_name self.block_blob_service = BlockBlobService(connection_string=connection_string) def upload(self, camera_id, base_name, extension , data): """ Upload data to blob storage. :param str camera_id: The ID of the camera that took the image. :param str base_name: The base filename of the file in storage, without extension. The camera module provides this, and it should be a string representation of the date and time at which the image was captured, such as 2019-06-15T13:45:30.459Z :param str extension: The file extension specifying the type of image, such as "jpg", "png", etc. The leading dot character should not be included. :param bytes data: The content of the blob to be uploaded. """ blob_name = camera_id + '/' + base_name + '.' + extension self.block_blob_service.create_blob_from_bytes(self.container_name, blob_name, data)
def test_account_sas_with_question_mark_prefix(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange token = '?' + self.account.generate_shared_access_signature( Services.BLOB, ResourceTypes.OBJECT + ResourceTypes.CONTAINER, AccountPermissions.READ + AccountPermissions.WRITE + AccountPermissions.DELETE + AccountPermissions.CREATE, datetime.utcnow() + timedelta(hours=1), ) service = BlockBlobService(self.account_name, sas_token=token, is_emulated=self.settings.IS_EMULATED) data = b'shared access signature with read/write permission on blob' container_name = self.get_resource_name("container") blob_name = 'blob1.txt' try: # Act service.create_container(container_name) service.create_blob_from_bytes(container_name, blob_name, data) blob = service.get_blob_to_bytes(container_name, blob_name) # Assert self.assertIsNotNone(blob) self.assertEqual(data, blob.content) finally: service.delete_container(container_name)
class AzureStorage: def __init__(self, connectionString, container): self.BlobService = BlockBlobService(connection_string=connectionString) nameValue = UtilityHelper.connectStringToDictionary(connectionString) self.AccountName = nameValue['AccountName'] self.container = container def getBaseURL(self): return 'https://' + self.AccountName + '.blob.core.windows.net/' def uploadByLocalFile(self, localFullFileName, remoteBlobName): self.BlobService.create_blob_from_path(self.container, remoteBlobName, localFullFileName) blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName return blobURL def uploadByStream(self, streamData, remoteBlobName): self.BlobService.create_blob_from_stream(self.container, remoteBlobName, streamData) blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName return blobURL def uploadByBytes(self, bytesData, remoteBlobName): self.BlobService.create_blob_from_bytes(self.container, remoteBlobName, bytesData) blobURL = 'https://' + self.AccountName + '.blob.core.windows.net/' + self.container + '/' + remoteBlobName return blobURL def delete(self, blobName): self.BlobService.delete_blob(self.container, blobName) def copy(self, sourceBlobURL, targetBlobName): self.BlobService.copy_blob(self.container, targetBlobName, sourceBlobURL)
def update(self, area, selector, content_type, buffer): assert area is not None, 'area is none; should already be validated' area_config = config.load_area(area) storage_config = config.load_storage(area_config['storage']) area = area.lower() # httplib.HTTPConnection.debuglevel = 1 # http.client.HTTPConnection.debuglevel = 1 blob_service = BlockBlobService(account_name=storage_config['name'], account_key=storage_config['key1']) hash = base64.b64encode(hashlib.md5(buffer).digest()) content_settings = ContentSettings(content_md5=hash) if content_type is not None and len(content_type) > 0: content_settings.content_type = content_type blob_service.create_blob_from_bytes( area_config['container'], selector, buffer, content_settings=content_settings, validate_content=False ) return hash
def __submit_image_from_url(proof_type, url): """Uploads an image to an azure blob storage instance that is also accessible by the clerk's module. Seems like a strange architectural decision--the A2P API should accept the image data as part of the user's submission.""" # TODO: Find a better way to get original filename from DA. filename = "ProofOf%s" % proof_type try: orig_filename = re.findall(r"filename%3D(.*?)(&|$)", url)[0][0] filename += "_%s" % orig_filename except Exception: pass blob_service = BlockBlobService( account_name=a2p_config()['blob_account_name'], account_key=a2p_config()['blob_account_key']) image_body = requests.get(url).content blob_service.create_blob_from_bytes('attachments', filename, image_body) return { "fileName": filename, "blobName": filename, "size": len(image_body) }
def add_case(): case_data = request.get_json() case_data['created_at'] = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") case = CaseSchema().load(case_data) case_id = str(db.cases.insert_one(case.data).inserted_id) block_blob_service = BlockBlobService( account_name='cryemlab', account_key= 'yo97cwqrvQLDJuQcY9fwpJqRJAQ9wPl5moUGCGxCesdyyEaByLT6L+0lCYBMZ31AqbtIAekAI429+U6UzEC/Vg==' ) container_name = 'cases' block_blob_service.create_container(container_name) block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container) image_data = re.sub('^data:image/.+;base64,', '', case_data['image']) byte_data = base64.b64decode(image_data) block_blob_service.create_blob_from_bytes(container_name, case_id + '.jpg', byte_data) return '', 204
def prokka_request(request): form = ProkkaForm() if request.method == 'POST': form = ProkkaForm(request.POST, request.FILES) if form.is_valid(): seqids, name, other_files = form.cleaned_data prokka_request_object = ProkkaRequest.objects.create( user=request.user, seqids=seqids, status='Processing') if name is None: prokka_request_object.name = prokka_request.pk else: prokka_request_object.name = name prokka_request_object.save() container_name = 'prokka-{}'.format(prokka_request_object.pk) blob_client = BlockBlobService( account_name=settings.AZURE_ACCOUNT_NAME, account_key=settings.AZURE_ACCOUNT_KEY) blob_client.create_container(container_name) file_names = list() for other_file in request.FILES.getlist('other_files'): file_name = os.path.join(container_name, other_file.name) file_names.append(file_name) blob_client.create_blob_from_bytes( container_name=container_name, blob_name=other_file.name, blob=other_file.read()) prokka_request_object.other_input_files = file_names prokka_request_object.save() run_prokka.apply_async(queue='cowbat', args=(prokka_request_object.pk, ), countdown=10) return redirect('geneseekr:prokka_result', prokka_request_pk=prokka_request_object.pk) return render(request, 'geneseekr/prokka_request.html', {'form': form})
def blob(): account_name = config.STORAGE_ACCOUNT_NAME account_key = config.STORAGE_ACCOUNT_KEY block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) container_name = config.BLOB_CONTAINER_NAME block_blob_service.create_container(container_name) img = load_img("/share1/public/isic/images/2.NV/ISIC_0034320.jpg", target_size=(224, 224)) img = img_to_array(img) imgbytes = img.tobytes() block_blob_service.create_blob_from_bytes(container_name, "test1", imgbytes) ib = block_blob_service.get_blob_to_bytes(container_name, "test1").content print(imgbytes == ib) print("\nList blobs in the container") generator = block_blob_service.list_blobs(container_name) for blob in generator: print("\t Blob name: " + blob.name)
def _save(self, data: pd.DataFrame) -> None: blob_service = BlockBlobService(**self._credentials) blob_service.create_blob_from_bytes( container_name=self._container_name, blob_name=self._filepath, blob=data.to_json(**self._save_args).encode(self._encoding), **self._blob_from_bytes_args)
def uploadToBlobStorage(dataToUpload, customer, dataName, storage_account_name, storage_account_key): # Create a blob container for each customer blockblobService = BlockBlobService(account_name=storage_account_name, account_key=storage_account_key) blockblobService.create_container(customer) # Upload the data to the correct blob container, under the correct dataname blockblobService.create_blob_from_bytes(customer, dataName, dataToUpload) print('{} for {} has been uploaded'.format(dataName, customer))
def write_blob_from_bytes(self, sas_uri, blob_name, input_bytes): sas_service = BlockBlobService( account_name=self.get_account_from_uri(sas_uri), sas_token=self.get_sas_key_from_uri(sas_uri)) container_name = self.get_container_from_uri(sas_uri) sas_service.create_blob_from_bytes(container_name, blob_name, input_bytes) return sas_service.make_blob_url(container_name, blob_name, sas_token=self.get_sas_key_from_uri(sas_uri))
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') # checking for a POST request. if (req.method == "POST"): try: # check JSON body request that carries the encoded image base64 info req_body = req.get_json() logging.info(req_body) image_name = req_body['name'] image = req_body['image'].replace(' ', '+') #decoding base64 image from json body decoded_image = base64.b64decode(image) logging.info(decoded_image) # Storage connection string connect_str = os.getenv('StorageConnStr') accuntName = os.getenv('StorageName') accuntKey = os.getenv('StorageConnectionKey') BlobPermissions(read=False, add=False, create=False, write=False, delete=False, _str=None) logging.info("Successful connection to blob storage.") #upload to picture to blob storage block_blob_service = BlockBlobService(account_name=accuntName, account_key=accuntKey) container_name = 'machineimages' blob_name = image_name + '.jpeg' # Creating the blob block_blob_service.create_blob_from_bytes( container_name, blob_name, decoded_image, content_settings=ContentSettings(content_type='image/png')) logging.info("Successfull blob creating ") # Returning a succesful post request return func.HttpResponse(f"successful request") except ValueError as e: logging.error("Invalid json format " + str(e)) pass except Exception as err: logging.error("Something went wrong decoding json " + str(err)) pass except AzureException as ae: logging.error("Something went wrong with azure connection " + str(ae)) pass return func.HttpResponse( "Please pass a name on the query string or in the request body", status_code=400)
def guardarDarUrl(file, filemane): baseUrl = 'https://catalogo2018storage.blob.core.windows.net/pictures/' sas = '?sv=2017-07-29&ss=bf&srt=co&sp=rwdlac&se=2018-05-19T00:27:02Z&st=2018-04-01T16:27:02Z&spr=https,http&sig=iJy3%2BhD2JhuYvXTRfsXT2qTM2p08tfhNGAfb%2BG5YR6w%3D' # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService(account_name=config('ACCOUNT_NAME', default=''), account_key=config('ACCOUNT_KEY', default='')) # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_bytes('pictures', filemane, file) return baseUrl + filemane + sas
def __submit_image_from_url(url): blob_service = BlockBlobService( account_name='a2pca', account_key=__get_a2p_config()['blob_account_key']) image_body = requests.get(url).content filename = 'a2p_daupload_' + hashlib.sha224(image_body).hexdigest() blob_service.create_blob_from_bytes('attachments', filename, image_body) return { "fileName": filename, "blobName": filename, "size": len(image_body) }
def storage(host): """ Create blob using azurite. """ bbs = BlockBlobService( account_name=USERNAME, account_key=KEY, custom_domain=f"http://{host}/devstoreaccount1", ) bbs.create_container("data", timeout=1) bbs.create_blob_from_bytes("data", "root/a/file.txt", data) bbs.create_blob_from_bytes("data", "root/b/file.txt", data) yield bbs
def guardar_archivo(archivo, nombre): # PUNTO DE CONEXIÓN DE SERVICIO BLOB PRINCIPAL baseUrl = config('STORAGE_URL', default='') # Una firma de acceso compartido (SAS) es un identificador URI que concede derechos de acceso # limitados a recursos de Azure Storage sas = config('SAS', default='') # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService(account_name=config('ACCOUNT_NAME', default=''), account_key=config('ACCOUNT_KEY', default='')) # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_bytes('pictures', nombre, archivo) return baseUrl + nombre + sas
def extractFrames(pathOut, filepath): # Path to video file cap = cv2.VideoCapture(filepath) #Reducing the frames per second of the video to 2 cap.set(cv2.CAP_PROP_FPS, 2) # Used as counter variable x = 1 frameRate = cap.get(5) #frame rate numberOfPicturesPerSecond = 2 blockBlobService = BlockBlobService( account_name='stworkersafety', account_key= '7OyzTj7Y83+0/+DiuS9IVDoZcKrQ0pSjE4F4q8L/ltT+Dv4TbBXTSDrOu928L60SCzo7mq+P3fEv3B4aOL6Flw==' ) # start creating frames from video while (cap.isOpened()): frameId = cap.get(1) #current frame number ret, frame = cap.read() if (ret != True): break # in case frame matches a multiple of the frame, create image if frameId % math.floor(frameRate / numberOfPicturesPerSecond) == 0: logging.info("create cap" + str(x)) # convert frame to PIL image frame_conv = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) pilImage = Image.fromarray(frame_conv) #Calculate size = Height/2 * Width/2 size = (round(pilImage.size[0] / 2), round(pilImage.size[1] / 2)) #Resize using CV2 pilImage = pilImage.resize(size, Image.ANTIALIAS) imgByteArr = BytesIO() pilImage.save(imgByteArr, format='jpeg') #print(type(pilImage)) imgByteArr = imgByteArr.getvalue() # write image to blob for logging now = datetime.strftime(datetime.now(), "%Y%m%dT%H%M%S%Z") imageFileName = 'epm_stage/image' + str( int(x)) + "_img_" + now + ".jpg" #imageFileName= 'folder' + "/log/image" + str(int(x)) + "_img.png" blockBlobService.create_blob_from_bytes('videoblob', imageFileName, imgByteArr) #Write to local directory pilImage.save(os.path.join(pathOut, "image{:d}.jpg".format(x))) #cv2.imwrite(os.path.join(pathOut , "image{:d}.jpeg".format(x)),frame) # increment image x += 1
class AzureFs(object): def __init__(self, name): self.bbs = BlockBlobService(account_name=acname, account_key=key, endpoint_suffix='core.chinacloudapi.cn') self.cname = name def put(self, content): x = uuid.uuid1() nm = str(x) + '.ms' self.bbs.create_blob_from_bytes(self.cname, nm, content) return nm def read(self, fid): return self.bbs.get_blob_to_bytes(self.cname, fid).content
class AzureStorage(BlobStorage): """Azure storage provider that utilizes the Azure blob storage. Args: connection_string: See http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ for the connection string format. container_name: the name of the blob container in which all blobs are stored. """ def __init__(self, connection_string, container_name): self._service = BlockBlobService(connection_string=connection_string) if not self._service.exists(container_name): raise ValueError("Container does not exist: " + container_name) self._container_name = container_name def get_object(self, blob_name): blob = self._service.get_blob_to_text(self._container_name, blob_name) return json.loads(blob.content) def put_object(self, obj, blob_name): data = json.dumps(obj).encode("utf-8") self._service.create_blob_from_bytes(self._container_name, blob_name, data) return Blob(blob_name, len(data)) @contextlib.contextmanager def get_file(self, blob_name): try: stream = AzureBlobReader(self._service, self._container_name, blob_name) yield stream finally: stream.close() def put_file(self, fileobj, blob_name): self._service.create_blob_from_stream(self._container_name, blob_name, fileobj) size = fileobj.tell() return Blob(blob_name, size) def put_avro(self, schema, records, blob_name, codec='snappy'): writer = AzureBlobWriter(self._service, self._container_name, blob_name) fastavro.writer(writer, schema, records, codec) writer.close() size = writer.tell() return Blob(blob_name, size)
def save_image(image, output_location, output_filename, container_name, format="png", bbs=None): """ Given a PIL.Image (list of pixel values), save to requested filename - note that the file extension will determine the output file type, can be .png, .tif, probably others... """ if not bbs: bbs = BlockBlobService(account_name=config["account_name"], account_key=config["account_key"]) output_path = os.path.join(output_location, output_filename) blob_name = remove_container_name_from_blob_path(output_path, container_name) im_bytes = io.BytesIO() image.save(im_bytes, format=format) bbs.create_blob_from_bytes(container_name, blob_name, im_bytes.getvalue())
def upload_metadata(request): form = RunNameForm() if request.method == 'POST': form = RunNameForm(request.POST) if form.is_valid(): if not SequencingRun.objects.filter( run_name=form.cleaned_data.get('run_name')).exists(): sequencing_run, created = SequencingRun.objects.update_or_create( run_name=form.cleaned_data.get('run_name'), seqids=list()) else: sequencing_run = SequencingRun.objects.get( run_name=form.cleaned_data.get('run_name')) files = [ request.FILES.get('file[%d]' % i) for i in range(0, len(request.FILES)) ] container_name = sequencing_run.run_name.lower().replace('_', '-') blob_client = BlockBlobService( account_name=settings.AZURE_ACCOUNT_NAME, account_key=settings.AZURE_ACCOUNT_KEY) blob_client.create_container(container_name) for item in files: blob_client.create_blob_from_bytes( container_name=container_name, blob_name=item.name, blob=item.read()) if item.name == 'SampleSheet.csv': instance = DataFile(sequencing_run=sequencing_run, data_file=item) instance.save() with open( 'olc_webportalv2/media/{run_name}/SampleSheet.csv'. format(run_name=str(sequencing_run))) as f: lines = f.readlines() seqid_start = False seqid_list = list() for i in range(len(lines)): if seqid_start: seqid = lines[i].split(',')[0] seqid_list.append(seqid) if 'Sample_ID' in lines[i]: seqid_start = True SequencingRun.objects.filter(pk=sequencing_run.pk).update( seqids=seqid_list) return redirect('cowbat:upload_interop', sequencing_run_pk=sequencing_run.pk) return render(request, 'cowbat/upload_metadata.html', {'form': form})
def azure_store_image_from_stream(self, img_name, file_stream): BlockBlobService = self.connect_azure() Container_name = "open-pai" path = BlockBlobService.create_blob_from_bytes(Container_name, 'test/img/' + img_name, file_stream) print("Upload success!") return path
def vir_typer_upload(request, vir_typer_pk): vir_typer_project = get_object_or_404(VirTyperProject, pk=vir_typer_pk) vir_typer_samples = list( VirTyperRequest.objects.filter(project_name__pk=vir_typer_pk)) sample_names = list() for sample in vir_typer_samples: sample_names.append(str(sample.LSTS_ID)) if request.method == 'POST': seq_files = [ request.FILES.get('file[%d]' % i) for i in range(0, len(request.FILES)) ] if seq_files: container_name = VirTyperProject.objects.get( pk=vir_typer_pk).container_namer() blob_client = BlockBlobService( account_name=settings.AZURE_ACCOUNT_NAME, account_key=settings.AZURE_ACCOUNT_KEY) blob_client.create_container(container_name) for item in seq_files: blob_client.create_blob_from_bytes( container_name=container_name, blob_name=item.name, blob=item.read()) for sample in vir_typer_samples: # sample_name = '{lsts}_{sn}'.format(lsts=str(sample.LSTS_ID), # sn=str(sample.sample_name)) for seq_file in seq_files: if str(sample.LSTS_ID) in str(seq_file): vir_files = VirTyperFiles(sample_name_id=sample.pk, sequence_file=seq_file) vir_files.save() vir_typer_project.status = 'Processing' vir_typer_project.save() run_vir_typer.apply_async(queue='cowbat', args=(vir_typer_pk, ), countdown=10) return redirect('vir_typer:vir_typer_home') return render( request, 'vir_typer/vir_typer_upload_sequences.html', { 'vir_typer_project': vir_typer_project, 'vir_typer_samples': vir_typer_samples, 'vir_typer_sample_names': sample_names })
def upldfile(): if request.method == 'POST': file = request.files['file'] if file and allowed_file(file.filename): filename = secure_filename(file.filename) app.logger.info('FileName: ' + filename) block_blob_service = BlockBlobService( account_name=app.config['AZURE_STORAGE_ACCOUNT'], account_key=app.config['AZURE_STORAGE_KEY']) block_blob_service.create_blob_from_bytes('doc', filename, file.read()) # updir = os.path.join(basedir, 'upload/') # file.save(os.path.join(updir, filename)) # file_size = os.path.getsize(os.path.join(updir, filename)) return jsonify(name=filename, url='https://'+app.config['AZURE_STORAGE_ACCOUNT']+'.blob.core.windows.net/' \ +app.config['AZURE_STORAGE_CONTAINER']+'/'+filename)
class BlobHelper: def __init__(self, blob=None): account_name = os.environ["AzureStorageAccountName"] account_key = os.environ["AzureStorageAccountKey"] self.blob_service = BlockBlobService( account_name=account_name, account_key=account_key ) self.blob = blob def create_output_blob(self, destination_container_name): source_url = os.environ["StorageUrl"] + self.blob.name destination_blob_name = self.get_destination_blob_name() self.blob_service.copy_blob( container_name=destination_container_name, blob_name=destination_blob_name, copy_source=source_url, ) def get_destination_blob_name(self): blob_filename = self.blob.name.split("/")[1] datetime_str = datetime.today().strftime("%Y%m%d-%H%M%S") return f"{datetime_str}-{blob_filename}" def get_str_file(self, storage_container_name, storage_blob_name): compressed_file = io.BytesIO() self.blob_service.get_blob_to_stream(storage_container_name, storage_blob_name, compressed_file, max_connections=1) compressed_file.seek(0) compressed_gzip = gzip.GzipFile(fileobj=compressed_file) decompressed_file = compressed_gzip.read() compressed_file.close() compressed_gzip.close() file_string = decompressed_file.decode("utf-8-sig") return file_string def write_stream_file(self, storage_container_name, storage_blob_name, encoded_file): self.blob_service.create_blob_from_bytes(storage_container_name, storage_blob_name, encoded_file, max_connections=1)
class AzureStorage(Storage): def __init__(self, option=None): self.block_blob_service = \ BlockBlobService(account_name=settings.AZURE_STORAGE_ACCOUNT_NAME, account_key=settings.AZURE_STORAGE_ACCOUNT_KEY) self.block_blob_service.create_container( settings.AZURE_STORAGE_DEFAULT_CONTAINER) def _save(self, name, content): content.open() content_stream = content.read() self.block_blob_service.create_blob_from_bytes( 'media', name, content_stream, content_settings=(ContentSettings( content_type=content.file.content_type))) return name def _open(self, name, mode='rb'): extension_index = name.rfind('.') extension = '' if extension_index != -1: extension = name[extension_index:] tmp_file = tempfile.NamedTemporaryFile(suffix=extension, delete=False) self.block_blob_service.get_blob_to_stream( container_name=settings.AZURE_STORAGE_DEFAULT_CONTAINER, blob_name=name, stream=tmp_file, max_connections=2) tmp_file.seek(0) return File(tmp_file) def exists(self, name): generator = self.block_blob_service.list_blobs('media') for blob in generator: if name == blob.name: return True return False def url(self, name): return self.name
class BlobStorage(): def __init__(self, account_name: str, sas_token: str, container: str): self.block_blob_service = BlockBlobService(account_name=account_name, sas_token=sas_token) self.container = container def _read_stream_from_blob(func): def wrapped_func(self, file_name, args, *kwargs): with BytesIO() as stream: self.block_blob_service.get_blob_to_stream( self.container, file_name, stream) stream.seek(0) return func(self, stream, args, *kwargs) return wrapped_func def _write_stream_to_blob(func): def wrapped_func(self, file_name, args, *kwargs): with BytesIO() as stream: func(self, stream, args, *kwargs) self.block_blob_service.create_blob_from_bytes( self.container, file_name, stream.getvalue()) return wrapped_func def _write_string_to_blob(func): def wrapped_func(self, file_name, *args, **kwargs): with StringIO() as stream: func(self, stream, *args, **kwargs) self.block_blob_service.create_blob_from_text( self.container, file_name, stream.getvalue()) return wrapped_func @_write_stream_to_blob def write_df_to_parquet(self, file_name: str, df: pd.DataFrame): df.to_parquet(file_name, index=False, engine='pyarrow') @_write_string_to_blob def write_df_to_csv(self, file_name: str, df: pd.DataFrame, *args, **kwargs): df.to_csv(file_name, index=False, *args, **kwargs)
def upload_sequence_data(request, sequencing_run_pk): sequencing_run = get_object_or_404(SequencingRun, pk=sequencing_run_pk) check_uploaded_seqids(sequencing_run=sequencing_run) if request.method == 'POST': container_name = sequencing_run.run_name.lower().replace('_', '-') blob_client = BlockBlobService( account_name=settings.AZURE_ACCOUNT_NAME, account_key=settings.AZURE_ACCOUNT_KEY) blob_client.create_container(container_name) for i in range(0, len(request.FILES)): item = request.FILES.get('file[%d]' % i) blob_client.create_blob_from_bytes(container_name=container_name, blob_name=item.name, blob=item.read()) # return redirect('cowbat:cowbat_processing', sequencing_run_pk=sequencing_run.pk) return render(request, 'cowbat/upload_sequence_data.html', { 'sequencing_run': sequencing_run, })
def update_rescued(): data=request.data #PERSON_GROUP_ID="victims" d=db.ngo_data.find_one({'type':'safe'}) #headers={"Content-Type":"application/octet-stream"} #headers["Ocp-Apim-Subscription-Key"]="501f22c3797048d2a73ae58a83ea9069" #BASE_URL="https://australiaeast.api.cognitive.microsoft.com/face/v1.0/" #cf.BaseUrl.set(BASE_URL) #cf.Key.set("501f22c3797048d2a73ae58a83ea9069") #binurl="https://australiaeast.api.cognitive.microsoft.com/face/v1.0/detect?returnFaceId=true&returnFaceLandmarks=false" #res=requests.post(url=binurl,headers=headers,data=data) #js=json.loads(res.text) #face_ids = [d['faceId'] for d in js] #identified_faces = cf.face.identify(face_ids, PERSON_GROUP_ID) #person_list=cf.person.lists(PERSON_GROUP_ID) uid=len(d['rescued_urls'])+1 # cursor["facial"] = "https://rvsafeimages.blob.core.windows.net/imagescontainer/"+uid+'.'+format_ # ref.update_one({"user_id":user_id,"Disasterid":str(cursor["Disasterid"])},{"$set":cursor},upsert=False) block_blob_service = BlockBlobService(account_name='rvsafeimages', account_key='391TMmlvDdRWu+AsNX+ZMl1i233YQfP5dxo/xhMrPm22KtwWwwMmM9vFAJpJHrGXyBrTW4OoAInjHnby9Couug==') container_name ='imagescontainer' block_blob_service.create_blob_from_bytes(container_name,'safevictims'+str(uid)+"."+'jpg',data) #save to blob urls="https://rvsafeimages.blob.core.windows.net/imagescontainer/safevictims"+str(uid)+'.'+'jpg' rescued=[] try: d['rescued_urls'].append(urls) except: d['rescued_urls']=[urls] # for k in identified_faces: # i=k['candidates'] # if len(i)==0: # continue # for t in i: # if 'personId' in t: # for j in person_list: # if j['personId']==t['personId']: # rescued.append([j['name'],urls]) # rescued=rescued+d['rescued'] # d['rescued']=rescued db.ngo_data.update_one({"type":"safe","_id":ObjectId("5c3b5389d59b290b704c4012")},{"$set":d},upsert=False) return json.dumps({'status':200})
def upload_images(userid, disasterid, format_): data = request.get_data() ref = db.Victim # import pdb; pdb.set_trace() cursor = ref.find_one({'user_id': userid}) if len(list(cursor)) == 0: assert ("Upload Failed") if "num_files" not in cursor: cursor["num_files"] = 0 nums = int(cursor["num_files"]) #nums has to be set to 0. nums += 1 cursor["num_files"] = nums if "blobnames" not in cursor: files = [] else: files = cursor["blobnames"] try: files = literal_eval(files) except: pass uid = userid + str(nums) files.append(uid + "." + format_) cursor["blobnames"] = files ref.update_one( { "_id": cursor["_id"], "user_id": userid, "Disasterid": disasterid }, {"$set": cursor}, upsert=False) block_blob_service = BlockBlobService( account_name='rvsafeimages', account_key= '391TMmlvDdRWu+AsNX+ZMl1i233YQfP5dxo/xhMrPm22KtwWwwMmM9vFAJpJHrGXyBrTW4OoAInjHnby9Couug==' ) container_name = 'imagescontainer' block_blob_service.create_blob_from_bytes(container_name, uid + "." + format_, data) #save to blob return json.dumps({"status": 200})
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, container_name, prefix=None): # NOTE: Azure wants all paths to start with a slash prefix = "/{}".format(prefix.lstrip("/") if prefix else "") super().__init__(prefix=prefix) self.account_name = account_name self.account_key = account_key self.container_name = container_name self.conn = BlockBlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized") # XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking #self.log.warning("AzureTransfer is experimental and has not been thoroughly tested") def get_metadata_for_key(self, key): key = self.format_key_for_backend(key) return self._list_blobs(key)[0]["metadata"] def _metadata_for_key(self, key): return self._list_blobs(key)[0]["metadata"] def list_path(self, key): path = self.format_key_for_backend(key, trailing_slash=True) return self._list_blobs(path) def _list_blobs(self, path): self.log.debug("Listing path %r", path) items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata") result = [] for item in items: result.append({ "last_modified": item.properties.last_modified.isoformat(), "metadata": item.metadata, "name": self.format_key_from_backend(item.name), "size": item.properties.content_length, }) return result def delete_key(self, key): key = self.format_key_for_backend(key) self.log.debug("Deleting key: %r", key) return self.conn.delete_blob(self.container_name, key) def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=None): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to) meta = self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to) if progress_callback: progress_callback(1, 1) return meta def get_contents_to_fileobj(self, key, fileobj_to_store_to, *, progress_callback=None): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) meta = self.conn.get_blob_to_file(self.container_name, key, fileobj_to_store_to) if progress_callback: progress_callback(1, 1) return meta def get_contents_to_string(self, key): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_bytes(self.container_name, key), self._metadata_for_key(key) def store_file_from_memory(self, key, memstring, metadata=None): key = self.format_key_for_backend(key) self.conn.create_blob_from_bytes(self.container_name, key, memstring, x_ms_meta_name_values=self.sanitize_metadata(metadata)) def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): key = self.format_key_for_backend(key) self.conn.create_blob_from_path(self.container_name, key, filepath,None) # x_ms_meta_name_values=self.sanitize_metadata(metadata)) def get_or_create_container(self, container_name): start_time = time.time() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time) return container_name
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, bucket_name, prefix=None): prefix = "{}".format(prefix.lstrip("/") if prefix else "") super().__init__(prefix=prefix) self.account_name = account_name self.account_key = account_key self.container_name = bucket_name self.conn = BlockBlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized, %r", self.container_name) def get_metadata_for_key(self, key): key = self.format_key_for_backend(key, remove_slash_prefix=True, trailing_slash=False) results = self._list_blobs(key) if not results: raise FileNotFoundFromStorageError(key) return results[0]["metadata"] def _metadata_for_key(self, key): return self._list_blobs(key)[0]["metadata"] def list_path(self, key, trailing_slash=True): # pylint: disable=arguments-differ # Trailing slash needed when listing directories, without when listing individual files path = self.format_key_for_backend(key, remove_slash_prefix=True, trailing_slash=trailing_slash) return self._list_blobs(path) def _list_blobs(self, path): self.log.debug("Listing path %r", path) if path: items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata") else: # If you give Azure an empty path, it gives you an authentication error items = self.conn.list_blobs(self.container_name, delimiter="/", include="metadata") results = [] for item in items: if not isinstance(item, BlobPrefix): results.append({ "last_modified": item.properties.last_modified, # Azure Storage cannot handle '-' so we turn them into underscores and back again "metadata": dict((k.replace("_", "-"), v) for k, v in item.metadata.items()), "name": self.format_key_from_backend(item.name), "size": item.properties.content_length, }) return results def delete_key(self, key): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.log.debug("Deleting key: %r", key) try: return self.conn.delete_blob(self.container_name, key) except azure.common.AzureMissingResourceHttpError as ex: raise FileNotFoundFromStorageError(key) from ex def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=None): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to) try: self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to) except azure.common.AzureMissingResourceHttpError as ex: raise FileNotFoundFromStorageError(key) from ex if progress_callback: progress_callback(1, 1) return self._metadata_for_key(key) def get_contents_to_fileobj(self, key, fileobj_to_store_to, *, progress_callback=None): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.log.debug("Starting to fetch the contents of: %r", key) try: self.conn.get_blob_to_stream(self.container_name, key, fileobj_to_store_to) except azure.common.AzureMissingResourceHttpError as ex: raise FileNotFoundFromStorageError(key) from ex if progress_callback: progress_callback(1, 1) return self._metadata_for_key(key) def get_contents_to_string(self, key): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.log.debug("Starting to fetch the contents of: %r", key) try: blob = self.conn.get_blob_to_bytes(self.container_name, key) return blob.content, self._metadata_for_key(key) except azure.common.AzureMissingResourceHttpError as ex: raise FileNotFoundFromStorageError(key) from ex def store_file_from_memory(self, key, memstring, metadata=None): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.conn.create_blob_from_bytes(self.container_name, key, memstring, metadata=self.sanitize_metadata(metadata, replace_hyphen_with="_")) def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.conn.create_blob_from_path(self.container_name, key, filepath, metadata=self.sanitize_metadata(metadata, replace_hyphen_with="_")) def get_or_create_container(self, container_name): start_time = time.monotonic() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.monotonic() - start_time) return container_name