def __perform_upload(self, clip: Clip, blob: BlobClient) -> bool:
try:
with open(clip.path, "rb") as data:
blob.upload_blob(data)
return True
except:
return False
def upload(globpath, container, sas_token_env, storage_account_uri):
try:
sas_token_env = sas_token_env
sas_token = os.getenv(sas_token_env)
if sas_token is None:
getLogger().error(
"Sas token environment variable {} was not defined.".format(
sas_token_env))
return 1
files = glob(globpath, recursive=True)
for infile in files:
blob_name = get_unique_name(infile, os.getenv('HELIX_WORKITEM_ID'))
getLogger().info("uploading {}".format(infile))
blob_client = BlobClient(blob_url=storage_account_uri,
container=container,
blob=blob_name,
credential=sas_token)
with open(infile, "rb") as data:
blob_client.upload_blob(data,
blob_type="BlockBlob",
content_settings=ContentSettings(
content_type="application/json"))
getLogger().info("upload complete")
except Exception as ex:
getLogger().error('{0}: {1}'.format(type(ex), str(ex)))
getLogger().error(format_exc())
return 1
def df_to_blob(
df: pd.DataFrame,
blob_client: BlobClient,
pandas_kwargs: Optional[Dict[str, Any]] = None,
overwrite: bool = False,
) -> None:
"""Upload a pandas DataFrame and store it into a blob."""
# check for kwargs
if not pandas_kwargs:
pandas_kwargs = {}
# check the file extension
extension = Path(blob_client.blob_name).suffix
# make DataFrame and upload
if extension in [".csv", ".txt", ".json"]:
string_io = io.StringIO()
if extension in [".csv", ".txt"]:
df.to_csv(string_io, **pandas_kwargs)
if extension == ".json":
df.to_json(string_io, **pandas_kwargs)
return blob_client.upload_blob(string_io.getvalue(),
overwrite=overwrite)
if extension in [".xls", ".xlsx"]:
bytes_io = io.BytesIO()
df.to_excel(bytes_io, **pandas_kwargs)
return blob_client.upload_blob(bytes_io.getvalue(),
overwrite=overwrite)
raise TypeError(f"{extension} files are not yet supported.")
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
image = req.files['image'].read()
image_name = req.files["image"].filename
board_id = req.form["board_id"]
logging.info("Image was read.")
time_local_timezone = datetime.datetime.utcnow().strftime(
"%Y-%m-%d-%H-%M-%S")
blob_client = BlobClient(
account_url="https://adlsmessageboard.blob.core.windows.net",
container_name="$web",
blob_name=f"images/{board_id}/{time_local_timezone}|{image_name}",
credential=os.environ["blob_key"])
blob_client.upload_blob(image, blob_type="BlockBlob", overwrite=True)
logging.info("Image was uploaded to BLOB.")
requests.post(
f"https://funcapp-messageboard.azurewebsites.net/api/azfunc_refreshBoard?board_id={board_id}"
)
return func.HttpResponse("Foto wurde hochgeladen.", status_code=200)
def __putFile(self, df: pd.DataFrame, blob_client: BlobClient) -> dict:
self._logger.info("Putting file: %s", blob_client.blob_name)
with io.StringIO() as buf:
df.to_csv(buf, index=False)
buf.seek(0)
# upload_blob can only take bytesio or file io, not stringio
with io.BytesIO(buf.read().encode('utf8')) as byte_buf:
blob_client.upload_blob(byte_buf, overwrite=True)
def test_upload_file_with_leading_slash(storage_account, sas_token):
client = BlobClient(
f"https://{storage_account}.blob.core.windows.net/",
credential=sas_token,
container=CONTAINER_NAME,
blob="/folder1/nestedfolder2/nestedfolder3/helloworld.txt",
)
with open("/mnt/c/code/noelbundick/storage-sdk-tests/data/loremipsum.txt",
"rb") as data:
client.upload_blob(data, overwrite=True)
assert "//folder1" not in client.url
decoded_url = unquote(client.url)
assert "//folder1" not in decoded_url
def upload_blob(container_name, blob_name, blob_data):
blob = BlobClient(env.REPORTS_STGACCT_URI,
container_name,
blob_name,
credential=CREDENTIALS)
response = blob.upload_blob(blob_data, overwrite=True)
return response
def upload_file(self, location, filename):
if '/' in filename:
tgt_filename = filename.split('/')[1]
else:
tgt_filename = filename
tgt_blob = BlobClient(
self.blob_service_client.url,
container_name=self.settings.storage_container,
blob_name=os.path.join(location, tgt_filename),
credential=self.sas_token
)
with open(filename, "rb") as data:
response = tgt_blob.upload_blob(data, overwrite=True)
print("upload result: ", response["error_code"])
if response["error_code"] is None:
return messages.message["ok"]
else:
return messages.message["upload_failed"]
def main(req: func.HttpRequest, enterMessageTemplateBlob: func.InputStream,
showMessageTemplateBlob: func.InputStream) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
board_id = dict(req.form)["board_id"]
if not board_id:
return func.HttpResponse(
f"Hello, please pass a board_id in the request body.")
# Verfiy name for forbidden characters
forbidden_character_list = ["/", "\\"]
if any(f in board_id for f in forbidden_character_list):
return func.HttpResponse(
"Der Name enthält ungültige Zeichen. Bitte nur Buchstaben, Zahlen und Leerzeichen verwenden."
)
# Check if board_id already exists
connection_str = os.environ["blobConnectionString"]
blob_service_client = BlobServiceClient.from_connection_string(
connection_str)
container_name = "$web"
container_client = blob_service_client.get_container_client(container_name)
blob_list = container_client.list_blobs(name_starts_with="live_webpages/")
existing_boards_list = list(
set([b["name"].split("/")[1] for b in blob_list]))
if board_id in existing_boards_list:
return func.HttpResponse(
"Dieser Name wird bereits verwendet, bitte einen anderen auswählen."
)
logging.info("Board name is valid.")
# Render both templates
enter_string_raw = str(enterMessageTemplateBlob.read())
show_string_raw = str(showMessageTemplateBlob.read())
for x in ["\\n", "\\r", "b'"]:
enter_string_raw = enter_string_raw.replace(x, "")
show_string_raw = show_string_raw.replace(x, "")
enter_template = Template(enter_string_raw)
show_template = Template(show_string_raw)
rendered_enter_template = enter_template.render(
board_id=board_id,
iframe_page=
f"https://adlsmessageboard.z1.web.core.windows.net/live_webpages/{board_id}/show_messages.html"
)
rendered_show_template = show_template.render(board_id=board_id,
content_items=[])
logging.info('Templates were processed.')
#Write finished webpages to BLOB
blob_client_enter = BlobClient(
account_url="https://adlsmessageboard.blob.core.windows.net",
container_name="$web",
blob_name=f"live_webpages/{board_id}/enter_message.html",
credential=os.environ["blob_key"])
blob_client_show = BlobClient(
account_url="https://adlsmessageboard.blob.core.windows.net",
container_name="$web",
blob_name=f"live_webpages/{board_id}/show_messages.html",
credential=os.environ["blob_key"])
logging.info('BLOB client was created.')
blob_client_enter.upload_blob(rendered_enter_template,
blob_type="BlockBlob",
overwrite=True,
content_settings=ContentSettings(
content_type="text/html; charset=utf-8"))
blob_client_show.upload_blob(rendered_show_template,
blob_type="BlockBlob",
overwrite=True,
content_settings=ContentSettings(
content_type="text/html; charset=utf-8"))
enter_page_url = f"https://adlsmessageboard.z1.web.core.windows.net/live_webpages/{board_id}/enter_message.html"
show_page_url = f"https://adlsmessageboard.z1.web.core.windows.net/live_webpages/{board_id}/show_messages.html"
response_string = f"Die Websiten wurden erstellt!\n Eingabe-Website: {enter_page_url}\n Anzeige-Website: {show_page_url}"
return func.HttpResponse(response_string)
blob_client_train = BlobClient(account_url="REPLACE",
container_name=container_name,
blob_name=local_file_name_train,
credential="REPLACE")
blob_client_forecast = BlobClient(account_url="REPLACE",
container_name=container_name,
blob_name=local_file_name_forecast,
credential="REPLACE")
blob_client_test = BlobClient(account_url="REPLACE",
container_name=container_name,
blob_name=local_file_name_test,
credential="REPLACE")
# Upload the created file
with open(upload_file_path_train, "rb") as data:
blob_client_train.upload_blob(data, overwrite=True)
print("\nUploading to Azure Storage as blob:\n\t" + local_file_name_train)
# blob_client_meta.upload_blob(meta_data, overwrite = True)
# print("\nUploading to Azure Storage as blob:\n\t" + local_file_name_meta)
with open(upload_file_path_forecast, "rb") as data:
blob_client_forecast.upload_blob(data, overwrite=True)
print("\nUploading to Azure Storage as blob:\n\t" +
local_file_name_forecast)
with open(upload_file_path_test, "rb") as data:
blob_client_test.upload_blob(data, overwrite=True)
print("\nUploading to Azure Storage as blob:\n\t" + local_file_name_test)
# Add section for deletion here.
def main(event: func.EventGridEvent):
event_data = event.get_json()
symbol = event_data.get('symbol')
statement = event_data.get('statement')
logging.info(f'Statement: {statement}\nSymbol: {symbol}')
# Define URL parameters for request to Alpha Vantage API
params = {'apikey': API_KEY, 'function': statement, 'symbol': symbol}
response = r.get(url=BASE_URL, params=params)
try:
data = response.json()
except json.JSONDecodeError as e:
logging.error(f'An error occurred while parsing response data.\n{e}')
raise e
else:
a = data.get('annualReports', []) # Annual data
q = data.get('quarterlyReports', []) # Get quarterly data
# Upload annual data
try:
if len(a) > 0:
target_blob_annual = BlobClient(
account_url=DL_ACCOUNT_URL,
container_name='alpha-vantage',
blob_name=
f'financial_statements/{statement}/annual/{symbol}.json',
credential=DL_KEY)
target_blob_annual.upload_blob(json.dumps(a), overwrite=True)
logging.info('Succesfully uploaded annual data.')
else:
logging.info('No annual data present')
except Exception as e:
logging.error(
f'An error occurred while uploading data to Data Lake.\n{e}')
raise e
# Upload quarterly data
try:
# Only upload when data is present
if len(q) > 0:
target_blob_quarterly = BlobClient(
account_url=DL_ACCOUNT_URL,
container_name='alpha-vantage',
blob_name=
f'financial_statements/{statement}/quarterly/{symbol}.json',
credential=DL_KEY)
target_blob_quarterly.upload_blob(json.dumps(q), overwrite=True)
logging.info('Succesfully uploaded quarterly data.')
else:
logging.info('No quarterly data present.')
except Exception as e:
logging.error(
f'An error occurred while uploading data to Data Lake.\n{e}')
raise e
def main(req: func.HttpRequest,
showMessageTemplateBlob: func.InputStream) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
board_id = req.params["board_id"]
# Check if board_id already exists
connection_str = os.environ["blobConnectionString"]
blob_service_client = BlobServiceClient.from_connection_string(
connection_str)
container_name = "$web"
container_client = blob_service_client.get_container_client(container_name)
blob_list = container_client.list_blobs(name_starts_with="live_webpages/")
existing_boards_list = list(
set([b["name"].split("/")[1] for b in blob_list]))
if board_id not in existing_boards_list:
return func.HttpResponse("Dieses Board existiert nicht.")
logging.info("Board name is valid.")
# Read all contents for this board from storage
table_service = TableService(account_name="adlsmessageboard",
account_key=os.environ["blob_key"])
min_datetime_for_messages = (
datetime.datetime.utcnow() -
datetime.timedelta(hours=24)).strftime("%Y-%m-%dT%H:%M:%SZ")
messages_today = table_service.query_entities(
"messages",
filter=
f"PartitionKey eq '{board_id}' and Timestamp gt datetime'{min_datetime_for_messages}'",
select="sender_name, timestamp, message_text")
message_list = [m for m in messages_today]
for m in message_list:
m.pop("etag")
message_texts = [d["message_text"] for d in message_list]
message_senders = [d["sender_name"] for d in message_list]
message_timestamps = [d["timestamp"] for d in message_list]
message_list_unique = []
distinct_message_texts = list(set(message_texts))
for t in distinct_message_texts:
messages_this_sender = [
d for d in message_list if d["message_text"] == t
]
text_sender = max([d["sender_name"] for d in messages_this_sender])
text_timestamp = max([d["timestamp"] for d in messages_this_sender])
message_list_unique.append({
"message_text": t,
"sender_name": text_sender,
"timestamp": text_timestamp
})
message_list = sorted(message_list_unique, key=itemgetter("timestamp"))
#Read images from storage
connection_str = os.environ["blobConnectionString"]
blob_service_client = BlobServiceClient.from_connection_string(
connection_str)
container_name = "$web"
container_client = blob_service_client.get_container_client(container_name)
blob_list = container_client.list_blobs(
name_starts_with=f"images/{board_id}/")
existing_images = sorted([b["name"] for b in blob_list], reverse=True)
num_images = min(len(existing_images), 3)
images_to_show = existing_images[0:num_images + 1]
images_full_paths = [
f"https://adlsmessageboard.z1.web.core.windows.net/" + f
for f in images_to_show
]
image_items = [{"image_path": p} for p in images_full_paths]
# Render template
show_string_raw = str(showMessageTemplateBlob.read())
for x in ["\\n", "\\r", "b'"]:
show_string_raw = show_string_raw.replace(x, "")
show_template = Template(show_string_raw)
rendered_show_template = show_template.render(board_id=board_id,
content_items=message_list,
image_items=image_items)
logging.info('Template was processed.')
#Write finished webpage to BLOB
blob_client_show = BlobClient(
account_url="https://adlsmessageboard.blob.core.windows.net",
container_name="$web",
blob_name=f"live_webpages/{board_id}/show_messages.html",
credential=os.environ["blob_key"])
logging.info('BLOB client was created.')
blob_client_show.upload_blob(rendered_show_template,
blob_type="BlockBlob",
overwrite=True,
content_settings=ContentSettings(
content_type="text/html; charset=utf-8"))
return func.HttpResponse("Board wurde geupdatet.")
def uploadResults(funcData: dict, resultsBlobClient: BlobClient, results_obj: dict):
try:
results_string = json.dumps(results_obj, default=default_json_encoder)
resultsBlobClient.upload_blob(results_string, overwrite=True)
except Exception as _e:
raise RuntimeError('Exiting due to unable to upload results file: {:s}. Error message: {:s}'.format(json.dumps(funcData), str(_e))) from _e
