def post(self):
data = feed_parser.parse_args()
image_file = data['avatarurl']
blob = BlobClient.from_connection_string(
conn_str=
"DefaultEndpointsProtocol=https;EndpointSuffix=core.windows.net;AccountName=image1689;AccountKey=LSKVztWeg3o9R4ufT6o9FtV4eW776/McWu+jaQWzjjrG4RnH8ztwyERNrVv8XIIHrdVnG4heorual1zOuVhjBg==",
container_name="images",
blob_name=str(uuid.uuid4().fields[-1])[:5] + ".jpg")
blob.upload_blob(image_file)
feed = FeedModel(data['name'], data['created_date'],
data['description'], blob.url)
feed.save_to_db()
return {"message": "Feed Created"}, 201
def create_blob_from_url(storage_connection_string,container_name):
# urls to fetch into blob storage
url_list = get_random_images()
for u in url_list:
print(f"copying file: {u} to blob storage...")
# Download file from url then upload blob file
r = requests.get(u, stream = True)
if r.status_code == 200:
r.raw.decode_content = True
block_blob_service = BlobClient.from_connection_string(conn_str=storage_connection_string, container_name=container_name, blob_name=get_filename_from_url(u))
block_blob_service.upload_blob(r.raw,overwrite=True)
def get_blob_size(account_name, container_name, blob_name):
try:
blob_client = BlobClient.from_connection_string(conn_str=azure_connection_string(account_name, connect_str_from_passwordstate), container_name=container_name, blob_name=blob_name)
blob_properties = blob_client.get_blob_properties()
length = blob_properties.size
return length
except Exception as ex:
if isinstance(ex, ResourceNotFoundError):
negative_length = -1
return negative_length
else:
print('Exception in get_blob_size for file ' + blob_name + ' :')
print(ex)
def registration_validation() -> Union[str, Response]:
session.pop('_flashes', None)
if request.method == 'POST':
email = session.get('email')
error = None
if 'code' not in request.form:
error = "Code is required."
if email is None:
error = "Email is required in Session."
try:
int(request.form['code'])
except ValueError:
error = "Enter a number."
if error is None:
code = int(request.form['code'])
if not check_code(code, email):
error = "Code is wrong."
if error is None:
session.clear()
if get_address_from_email(email) is not None:
session["installed"] = True
save_to_db(email)
username = get_username_from_email(email)
default_client = BlobClient.from_connection_string(BLOB_CONNECTION_STRING, PROFILE_PICTURES_CONTAINER,
'default-profile.png')
data = default_client.download_blob().readall()
user_client = BlobClient.from_connection_string(BLOB_CONNECTION_STRING, PROFILE_PICTURES_CONTAINER,
f"{username}.png")
user_client.upload_blob(data, overwrite=True)
response = make_response(redirect(url_for('main.gallery')))
access_token = create_access_token(identity={"email": email, "username": username})
refresh_token = create_refresh_token(identity={"email": email, "username": username})
set_access_cookies(response, access_token)
set_refresh_cookies(response, refresh_token)
return response
flash(error)
return render_template('auth/registration_validation.html')
def stop_blob_copy(self, container_name, blob_name):
source_blob = BlobClient.from_connection_string(
os.getenv("AZURE_STORAGE_CONNECTION_STRING"),
container_name, blob_name
)
try:
dest_blob = BlobClient.from_connection_string(
os.getenv("AZURE_STORAGE_CONNECTION_STRING"),
container_name, str(uuid.uuid4()) + "-" + blob_name
)
# Start the copy operation.
dest_blob.start_copy_from_url(source_blob.url)
# <Snippet_StopBlobCopy>
# Get the destination blob's properties to check the copy status.
properties = dest_blob.get_blob_properties()
copy_props = properties.copy
# Check the copy status. If the status is pending, abort the copy operation.
if (copy_props["status"] == "pending"):
dest_blob.abort_copy(copy_props["id"])
print("Copy operation " + copy_props["id"] + " has been aborted.")
# </Snippet_StopBlobCopy>
# Display the copy status
print("Copy status: " + copy_props["status"])
print("Copy progress: " + copy_props["progress"]);
print("Completion time: " + str(copy_props["completion_time"]));
print("Total bytes: " + str(properties.size));
except ResourceNotFoundError as ex:
print("ResourceNotFoundError: ", ex.message)
except ServiceRequestError as ex:
print("ServiceRequestError: ", ex.message)
def download_i3_gcd_file(out_dir):
container = ContainerClient.from_connection_string(
conn_str=CONN_STR, container_name=GCD_CONTAINER_NAME)
gcd_list = container.list_blobs()
for blob in gcd_list:
gcd_name = blob.name
out_file = out_dir.joinpath(gcd_name)
blob = BlobClient.from_connection_string(conn_str=CONN_STR,
container_name=GCD_CONTAINER_NAME,
blob_name=gcd_name)
with open(str(out_file), 'wb') as blob_file:
blob_data = blob.download_blob()
blob_data.readinto(blob_file)
return gcd_name
def get_images_from_container(container_name):
connect_str = "#ENTER YOUR CONNECTION STRING HERE"
container = ContainerClient.from_connection_string(
conn_str=connect_str, container_name=container_name)
blob_list = container.list_blobs()
for blob in blob_list:
blob_instance = BlobClient.from_connection_string(
conn_str=connect_str,
container_name=container_name,
blob_name=blob.name)
with open(f"./static/images/{blob.name}", "wb") as my_blob:
blob_data = blob_instance.download_blob()
blob_data.readinto(my_blob)
def download(file):
parser = ConfigParser()
parser.read('config.ini')
connection_string = parser.get('AZURE', 'STORAGE_CONNECTION_STRING')
container_name = parser.get('AZURE', 'STORAGE_CONTAINER')
blob = BlobClient.from_connection_string(connection_string,
container=container_name,
blob=file)
local_file = "./files/{}".format(file)
with open(local_file, "wb") as my_blob:
blob_data = blob.download_blob()
my_blob.write(blob_data.content_as_bytes())
def file_upload_to_blob(account_name, container_name, blob_name, dfs_path):
try:
blob_client = BlobClient.from_connection_string(
conn_str=azure_connection_string(account_name,
connect_str_from_passwordstate),
container_name=container_name,
blob_name=blob_name)
blob_client.upload_blob(dfs_path)
return True
except Exception as ex:
if isinstance(ex, ResourceExistsError):
pass
else:
print('Exception in file_upload_to_blob:')
print(ex)
def upload(connection_string, container, filepath, blobname, headers):
LOGGER.info("uploading to blob")
blob = BlobClient.from_connection_string(connection_string,
container_name=container,
blob_name=blobname,
headers=headers,
connection_timeout=5,
retry_total=3)
try:
with open(filepath, 'rb') as data:
blob.upload_blob(data, overwrite=False)
except AzureError as error:
LOGGER.exception(error)
def get_blob(blobname: str) -> Optional[dict]:
blob = BlobClient.from_connection_string(
conn_str=AZURE_STORAGE_CONNECTION_STRING,
container_name=AZURE_STORAGE_CONTAINER,
blob_name=blobname)
if blob.exists():
tmp_bytes = BytesIO()
tmp_bytes.write(blob.download_blob().readall())
tmp_bytes.seek(0)
data = load(tmp_bytes)
if isinstance(data, str):
return loads(data)
else:
return data
return None
def _create_client(self, filename: str) -> BlobClient:
"""Creates blob client object
:param filename: name of file to open
:return: blob client object
"""
# ensure last character on path is '/' to allow for filename appending
if self.path[-1] != '/':
self.path = self.path + '/'
file_to_open = self.path + filename
return BlobClient.from_connection_string(conn_str=self._connection_string,
container_name=self._container,
blob_name=file_to_open)
def uploadplot():
plot = request.files['plot']
if plot.filename != '':
# SE CORRER NOUTRA MAQUINA
plot.save(os.path.join(app.config['UPLOAD_FOLDER'], plot.filename))
blob = BlobClient.from_connection_string(
conn_str=config('CONNECT_STRING'),
container_name=request.form.get('container'),
blob_name=plot.filename)
with open(os.path.join(app.config['UPLOAD_FOLDER'], plot.filename),
"rb") as data:
blob.upload_blob(data)
return str("ficheiro guardado")
return str("ficheiro nao guardado")
def main():
host = "localhost"
port = 6379
cache = redis.Redis(host, port)
blob_connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" \
"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr" \
"/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" \
"QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;"
container_name = "my-files"
container_client = ContainerClient.from_connection_string(
blob_connection_string, container_name)
try:
container_client.delete_container()
except ResourceNotFoundError:
pass
container_client.create_container()
blob_name = "my-file.txt"
cache.delete(blob_name)
blob_client = BlobClient.from_connection_string(blob_connection_string,
container_name, blob_name)
print("Uploading blob...")
blob_client.upload_blob("Hello!")
for i in range(4):
start = time.perf_counter()
data = cache.get(blob_name)
if not data:
print("Cache miss.")
blob = blob_client.download_blob()
data = blob.readall()
cache.set(blob_name, data)
print(f"File contents: {data}")
else:
print("Cache hit.")
print(f"File contents: {data}")
print(f"Iteration {i + 1} time: {time.perf_counter() - start:0.4f}s")
def copy_from_s3(
self,
s3_client,
s3_bucket_name: str,
s3_object_key: str,
target_blob_name: str
):
'''Copy an object from Amazon S3 to an Azure Storage Account
This will overwrite the contents of the target file if it already exists.
'''
connection_string = (
f"DefaultEndpointsProtocol=https;"
f"AccountName={self.sa_name};"
f"AccountKey={self.sa_key};"
"EndpointSuffix=core.windows.net"
)
image_blob = BlobClient.from_connection_string(
conn_str=connection_string,
container_name=self.container_name,
blob_name=target_blob_name,
blob_type=BlobType.PageBlob,
)
file_size_response = s3_client.head_object(Bucket=s3_bucket_name, Key=s3_object_key)
file_size = file_size_response['ContentLength']
url = s3_client.generate_presigned_url(
'get_object',
Params={'Bucket': s3_bucket_name, 'Key': s3_object_key},
)
image_blob.create_page_blob(file_size)
# max size we can copy in one go is 4 mebibytes. Split the upload in steps with max size of
# 4 MiB
copy_step_length = 4 * 1024 * 1024
offset = 0
while offset < file_size:
remaining = file_size - offset
actual_cp_bytes = min(copy_step_length, remaining)
image_blob.upload_pages_from_url(
source_url=url,
offset=offset,
length=actual_cp_bytes,
source_offset=offset,
)
offset += actual_cp_bytes
def main(event: func.EventGridEvent):
event_type = event.event_type
event_subject = event.subject
containername = event_subject.split("/")[-3]
# credentials needed
connection_string = os.getenv('AZURE_CONNECTION_STRING')
container_name = os.getenv('ContainerName')
# Only blob creations at container would trigger this function
if event_type == "Microsoft.Storage.BlobCreated" and containername == container_name:
filename = event_subject.split("/")[-1]
tablename = gettablename(
filename=filename) + datetime.now().strftime("%H%M%S")
table_service = TableService(connection_string=connection_string)
table_service.create_table(table_name=tablename)
# Download the blob data
blob = BlobClient.from_connection_string(conn_str=connection_string,
container_name=container_name,
blob_name=filename)
blob_data = blob.download_blob().readall()
json_file_content = blob_data.decode('utf8').replace("'", '"')
jlist = json.loads(json_file_content)
# The partition key might be changed later. This is only for DEVELOPMENT purpose
partition_key = tablename
row = 1
batch_list = []
for jdict in jlist:
if 'Cruise_ID' in jdict:
partition_key = jdict['Cruise_ID']
task = {'PartitionKey': partition_key, 'RowKey': str(row)}
for key in jdict:
keyVal = key.replace(' ', '')
if keyVal == '':
continue
task[keyVal] = jdict[key]
batch_list.append(task)
row += 1
seperated_batch_list = list_of_groups(init_list=batch_list,
childern_list_len=50)
for children_list in seperated_batch_list:
batch = TableBatch()
for task in children_list:
batch.insert_or_replace_entity(task)
table_service.commit_batch(table_name=tablename, batch=batch)
def getBlobFromAzure():
connection_string, container_name = get_credentials()
url = urlparse(filecontent)
filename = os.path.basename(url.path)
pre, ext = os.path.splitext(filename)
updated_filename = tidyname(pre) + ".json"
blob = BlobClient.from_connection_string(conn_str=connection_string,
container_name=container_name,
blob_name=updated_filename)
blob_data = blob.download_blob().readall()
json_file_content = blob_data.decode('utf8').replace("'", '"')
jlist = json.loads(json_file_content)
return jlist
def etl():
# request existing data from webservices endpoint
api_key = os.environ["API_KEY"]
conn_str = os.environ["AzureWebJobsStorage"]
container_name = os.environ["CONTAINER_NAME"]
file_name = os.getenv("FILE_NAME", "ita_taxonomy_labels.json")
url = f"https://api.trade.gov/ita_taxonomies/search?api_key={api_key}&size=-1"
req = requests.get(url)
json_response = req.json()
logging.info(f"Loaded {json_response['total']} records from {url}")
# upload results JSON array to Blob
blob = BlobClient.from_connection_string(
conn_str=conn_str, container_name=container_name, blob_name=file_name
)
blob.upload_blob(json.dumps(json_response["results"]), overwrite=True)
logging.info(f"Uploaded JSON to {file_name} blob in {container_name} container")
def get_blob_from_azure(
local_file_name: str,
full_path_to_file: str,
container_name: Optional[str] = AZURE_SYNKER_BLOB_CONTAINER,
connection_string: Optional[str] = AZURE_SYNKER_BLOB_CNX_STRING,
) -> None:
blob = BlobClient.from_connection_string(
conn_str=connection_string,
container_name=container_name,
blob_name=full_path_to_file,
)
with open(local_file_name, "wb") as my_blob:
stream = blob.download_blob()
data = stream.readall()
my_blob.write(data)
def upload_to_azure(
full_path_to_file: str,
local_file_name: str,
container_name: Optional[str] = AZURE_SYNKER_BLOB_CONTAINER,
connection_string: Optional[str] = AZURE_SYNKER_BLOB_CNX_STRING,
) -> str:
blob = BlobClient.from_connection_string(
conn_str=connection_string,
container_name=container_name,
blob_name=full_path_to_file,
)
with open(local_file_name, "rb") as data:
blob.upload_blob(data)
return f"https://{AZURE_SYNKER_ACCOUNT_NAME}.blob.core.windows.net/{container_name}/{full_path_to_file}"
def kategori_ekle(request):
if request.method == 'GET':
getkategoris = Category.objects.all()
return render(request=request, template_name='spor_kategori_ekle.html')
if request.method == 'POST':
trkateisim = request.POST.get('trkateisim')
trkateaciklama = request.POST.get('trkateaciklama')
enkateisim = request.POST.get('enkateisim')
enkateaciklama = request.POST.get('enkateaciklama')
filez = request.FILES['file']
if trkateisim:
fs = FileSystemStorage()
filename = fs.save(filez.name, filez)
num = random.randrange(1, 10**3)
newBasename = "sporKategory" + trkateisim + str(num) + ".jpg"
newname = os.path.join(path, newBasename)
oldname = os.path.join(path, filename)
os.rename(oldname, newname)
blob = BlobClient.from_connection_string(
container_name="spor-kategori",
conn_str=string,
blob_name=newBasename)
with open(newname, "rb") as data:
blob.upload_blob(data)
try:
getcategory = Category.objects.get(name_tr=trkateisim)
messages.error(
request,
"Bu Kategori zaten mevcut, baska kategori ekleyin")
except OSError:
messages.error(request, "Resim kaydedilemedi tekrar deneyin")
except Category.DoesNotExist:
savecate = Category(name_en=enkateisim,
name_tr=trkateisim,
explain_en=enkateaciklama,
explain_tr=trkateaciklama,
image=newBasename)
savecate.save()
messages.success(request, "Kategori Basari ile Kaydedildi")
os.remove(newname)
return render(request=request,
template_name='spor_kategori_ekle.html')
else:
messages.error(request, "Lutfen Turkce Kategori Ismi Belirleyin")
return render(request=request,
template_name='spor_kategori_ekle.html')
def download_from_azure(account_name: str,
account_key: str,
container_name: str,
blob_name: str,
file_name: str,
log: logging.Logger,
download_path: str = videos) -> Tuple:
"""Download file from Microsoft Azure.
Download file from Microsoft Azure and store it in videos folder.
Args:
account_name: Azure account name.
account_key: Azure account key.
container_name: Container from which blob needs to be downloaded.
blob_name: Blob to download from Microsoft Azure.
file_name: Filename for the downloaded file.
log: Logger object for logging the status.
download_path: Path (default: ./videos/) for saving file.
Returns:
Boolean value if the file is downloaded or not.
"""
# You can find the reference code here:
# https://pypi.org/project/azure-storage-blob/
try:
connection_string = generate_connection_string(account_name,
account_key)
blob = BlobClient.from_connection_string(conn_str=connection_string,
container_name=container_name,
blob_name=blob_name)
with open(os.path.join(download_path, f'{file_name}.mp4'),
'wb') as file:
data = blob.download_blob()
data.readinto(file)
log.info(f'File "{file_name}.mp4" downloaded from Microsoft Azure.')
if file_size(os.path.join(download_path,
f'{file_name}.mp4')).endswith('KB'):
log.error('Unusable file downloaded since file size is in KBs.')
return None, '[w] Unusable file downloaded.'
return True, os.path.join(download_path, f'{file_name}.mp4')
except Exception:
log.error('File download from Microsoft Azure failed because of poor '
'network connectivity.')
return None, '[e] Error while downloading file'
def main(req):
# return func.HttpResponse('[{"faceId":"6baaa70d-7fcf-4f36-9127-a35c914b28d1","faceRectangle":{"top":340,"left":202,"width":192,"height":139},"faceAttributes":{"age":33.0,"emotion":{"anger":0.0,"contempt":0.0,"disgust":0.0,"fear":0.0,"happiness":0.001,"neutral":0.986,"sadness":0.0,"surprise":0.012}}}]')
b64_img = req.params.get("b64_img")
if not b64_img:
try:
req_body = req.get_json()
except ValueError:
pass
else:
b64_img = req_body.get("b64_img")
# Validation
if b64_img and "data:image/png;base64," == b64_img[:22]:
# New File Name
file_name = f"{str(uuid4())}.png"
# Connection
blob = BlobClient.from_connection_string(
conn_str=CONN_STR,
container_name=CONTAINER,
blob_name=file_name,
)
# String to Bytes
bytes = base64.decodebytes(b64_img[22:].encode("utf-8"))
# Upload
blob.upload_blob(bytes)
# ML
res = get_facial_attributes(blob.url)
# cleanup
blob.delete_blob()
return func.HttpResponse(res)
# Error Message
else:
return func.HttpResponse(
"Please pass b64_img (png) on the query string or in the request body",
status_code=400,
)
def write_on_blob_storage(data):
data_bytes = data['base64Bytes']
conn_str = os.getenv('blob_storage_connection_string', None)
container_name = os.getenv('blob_storage_container', None)
blob_name = f"{uuid4()}.jpeg"
if conn_str and container_name:
# Create full Blob URL
x = conn_str.split(';')
image_url = f"{x[0].split('=')[1]}://{x[1].split('=')[1]}.{x[3].split('=')[1]}/{container_name}/{blob_name}"
data['image_url'] = image_url
# Upload data on Blob
blob_client = BlobClient.from_connection_string(
conn_str=conn_str,
container_name=container_name,
blob_name=blob_name)
content_settings = ContentSettings(content_type='image/jpeg')
blob_client.upload_blob(data_bytes, content_settings=content_settings)
return data
def get_blob_infos(connection_s: str, container_n: str, blob_n: str):
"""Get blobs information url
Arguments:
connection_s {str} -- an azure storage account connection string
container_n {str} -- a container within a storage account
blob_n {str} -- the name of the blob get url info from
"""
try:
blob_video = BlobClient.from_connection_string(
conn_str=connection_s,
container_name=container_n,
blob_name=blob_n)
blob_video_url = blob_video.url
logger.info(f'Blob URL:{blob_video_url}')
except:
logger.error(
"The blob you are trying to get info from probably does not exist."
)
def _initialise(self):
self.client: BlobClient = BlobClient.from_connection_string(
conn_str=self._connection_string,
container_name=self.container,
blob_name=self._path,
# retry_to_secondary=True,
connection_timeout=60,
max_block_size=8 * 1024 * 1024,
max_single_put_size=256 * 1024 * 1024,
min_large_block_upload_threshold=8 * 1024 * 1024 + 1)
self.account_name = self.client.account_name
self.target = self.client.primary_hostname
self.url = "{}://{}/{}/{}".format(
self.client.scheme,
self.client.primary_hostname,
quote(self.container),
quote(self._path, safe='~/'),
)
def post(self):
data = UserRegister.parser.parse_args()
if UserModel.find_by_username(data['username']):
return {
'message': "A user with that username already exists."
}, 400
image_file = data['avatarurl']
blob = BlobClient.from_connection_string(
conn_str=
"DefaultEndpointsProtocol=https;EndpointSuffix=core.windows.net;AccountName=image1689;AccountKey=LSKVztWeg3o9R4ufT6o9FtV4eW776/McWu+jaQWzjjrG4RnH8ztwyERNrVv8XIIHrdVnG4heorual1zOuVhjBg==",
container_name="images",
blob_name=str(uuid.uuid4().fields[-1])[:5] + ".jpg")
blob.upload_blob(image_file)
user = UserModel(data['username'], data['password'], data['email'],
blob.url)
user.save_to_db()
return {"message": "User Created!"}, 201
def _download_file_azure(self, ticket, fileurl):
tmp = fileurl.replace('azure://', '').split('/')
container_name = tmp[0]
blob_name = tmp[1]
ticket_path = os.path.join(bqueryd.INCOMING, ticket)
if not os.path.exists(ticket_path):
try:
os.makedirs(ticket_path)
except OSError as ose:
if ose == errno.EEXIST:
pass # different processes might try to create the same directory at _just_ the same time causing the previous check to fail
temp_path = os.path.join(bqueryd.INCOMING, ticket, blob_name)
if os.path.exists(temp_path):
self.logger.info("%s exists, skipping download" % temp_path)
self.file_downloader_progress(ticket, fileurl, 'DONE')
else:
self.logger.info(
"Downloading ticket [%s], container name [%s], blob name [%s]"
% (ticket, container_name, blob_name))
# Download blob
try:
fd, tmp_filename = tempfile.mkstemp(dir=bqueryd.INCOMING)
blob_client = BlobClient.from_connection_string(
conn_str=self.azure_conn_string,
container_name=container_name,
blob_name=blob_name)
download_stream = blob_client.download_blob()
with open(tmp_filename, 'wb') as fh:
fh.write(
download_stream.content_as_bytes(max_concurrency=1))
self._unzip_tmp_file(os.path.dirname(temp_path), tmp_filename)
finally:
if os.path.exists(tmp_filename):
os.remove(tmp_filename)
os.close(fd)
self.logger.debug('Download done %s azure://%s/%s', ticket,
container_name, blob_name)
self.file_downloader_progress(ticket, fileurl, 'DONE')
def blob_download(containerName, blobName):
"""
Download blobName from containerName
Output = a local version of blobName is saved to current directory
Return = name of local file, which is just blobName
"""
#
# download blob from container
#
blob = BlobClient.from_connection_string(conn_str=blob_connection_string,
container_name=containerName,
blob_name=blobName)
with open(blobName, "wb") as my_blob:
blob_data = blob.download_blob()
blob_data.readinto(my_blob)
return blobName
def auth_connection_string(self):
# [START auth_from_connection_string]
from azure.storage.blob import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
# [END auth_from_connection_string]
# [START auth_from_connection_string_container]
from azure.storage.blob import ContainerClient
container_client = ContainerClient.from_connection_string(
self.connection_string, container_name="mycontainer")
# [END auth_from_connection_string_container]
# [START auth_from_connection_string_blob]
from azure.storage.blob import BlobClient
blob_client = BlobClient.from_connection_string(
self.connection_string, container_name="mycontainer", blob_name="blobname.txt")
# [END auth_from_connection_string_blob]
# Get account information for the Blob Service
account_info = blob_service_client.get_account_information()
