def __perform_upload(self, clip: Clip, blob: BlobClient) -> bool: try: with open(clip.path, "rb") as data: blob.upload_blob(data) return True except: return False
def upload(globpath, container, sas_token_env, storage_account_uri): try: sas_token_env = sas_token_env sas_token = os.getenv(sas_token_env) if sas_token is None: getLogger().error( "Sas token environment variable {} was not defined.".format( sas_token_env)) return 1 files = glob(globpath, recursive=True) for infile in files: blob_name = get_unique_name(infile, os.getenv('HELIX_WORKITEM_ID')) getLogger().info("uploading {}".format(infile)) blob_client = BlobClient(blob_url=storage_account_uri, container=container, blob=blob_name, credential=sas_token) with open(infile, "rb") as data: blob_client.upload_blob(data, blob_type="BlockBlob", content_settings=ContentSettings( content_type="application/json")) getLogger().info("upload complete") except Exception as ex: getLogger().error('{0}: {1}'.format(type(ex), str(ex))) getLogger().error(format_exc()) return 1
def df_to_blob( df: pd.DataFrame, blob_client: BlobClient, pandas_kwargs: Optional[Dict[str, Any]] = None, overwrite: bool = False, ) -> None: """Upload a pandas DataFrame and store it into a blob.""" # check for kwargs if not pandas_kwargs: pandas_kwargs = {} # check the file extension extension = Path(blob_client.blob_name).suffix # make DataFrame and upload if extension in [".csv", ".txt", ".json"]: string_io = io.StringIO() if extension in [".csv", ".txt"]: df.to_csv(string_io, **pandas_kwargs) if extension == ".json": df.to_json(string_io, **pandas_kwargs) return blob_client.upload_blob(string_io.getvalue(), overwrite=overwrite) if extension in [".xls", ".xlsx"]: bytes_io = io.BytesIO() df.to_excel(bytes_io, **pandas_kwargs) return blob_client.upload_blob(bytes_io.getvalue(), overwrite=overwrite) raise TypeError(f"{extension} files are not yet supported.")
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') image = req.files['image'].read() image_name = req.files["image"].filename board_id = req.form["board_id"] logging.info("Image was read.") time_local_timezone = datetime.datetime.utcnow().strftime( "%Y-%m-%d-%H-%M-%S") blob_client = BlobClient( account_url="https://adlsmessageboard.blob.core.windows.net", container_name="$web", blob_name=f"images/{board_id}/{time_local_timezone}|{image_name}", credential=os.environ["blob_key"]) blob_client.upload_blob(image, blob_type="BlockBlob", overwrite=True) logging.info("Image was uploaded to BLOB.") requests.post( f"https://funcapp-messageboard.azurewebsites.net/api/azfunc_refreshBoard?board_id={board_id}" ) return func.HttpResponse("Foto wurde hochgeladen.", status_code=200)
def __putFile(self, df: pd.DataFrame, blob_client: BlobClient) -> dict: self._logger.info("Putting file: %s", blob_client.blob_name) with io.StringIO() as buf: df.to_csv(buf, index=False) buf.seek(0) # upload_blob can only take bytesio or file io, not stringio with io.BytesIO(buf.read().encode('utf8')) as byte_buf: blob_client.upload_blob(byte_buf, overwrite=True)
def test_upload_file_with_leading_slash(storage_account, sas_token): client = BlobClient( f"https://{storage_account}.blob.core.windows.net/", credential=sas_token, container=CONTAINER_NAME, blob="/folder1/nestedfolder2/nestedfolder3/helloworld.txt", ) with open("/mnt/c/code/noelbundick/storage-sdk-tests/data/loremipsum.txt", "rb") as data: client.upload_blob(data, overwrite=True) assert "//folder1" not in client.url decoded_url = unquote(client.url) assert "//folder1" not in decoded_url
def upload_blob(container_name, blob_name, blob_data): blob = BlobClient(env.REPORTS_STGACCT_URI, container_name, blob_name, credential=CREDENTIALS) response = blob.upload_blob(blob_data, overwrite=True) return response
def upload_file(self, location, filename): if '/' in filename: tgt_filename = filename.split('/')[1] else: tgt_filename = filename tgt_blob = BlobClient( self.blob_service_client.url, container_name=self.settings.storage_container, blob_name=os.path.join(location, tgt_filename), credential=self.sas_token ) with open(filename, "rb") as data: response = tgt_blob.upload_blob(data, overwrite=True) print("upload result: ", response["error_code"]) if response["error_code"] is None: return messages.message["ok"] else: return messages.message["upload_failed"]
def main(req: func.HttpRequest, enterMessageTemplateBlob: func.InputStream, showMessageTemplateBlob: func.InputStream) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') board_id = dict(req.form)["board_id"] if not board_id: return func.HttpResponse( f"Hello, please pass a board_id in the request body.") # Verfiy name for forbidden characters forbidden_character_list = ["/", "\\"] if any(f in board_id for f in forbidden_character_list): return func.HttpResponse( "Der Name enthält ungültige Zeichen. Bitte nur Buchstaben, Zahlen und Leerzeichen verwenden." ) # Check if board_id already exists connection_str = os.environ["blobConnectionString"] blob_service_client = BlobServiceClient.from_connection_string( connection_str) container_name = "$web" container_client = blob_service_client.get_container_client(container_name) blob_list = container_client.list_blobs(name_starts_with="live_webpages/") existing_boards_list = list( set([b["name"].split("/")[1] for b in blob_list])) if board_id in existing_boards_list: return func.HttpResponse( "Dieser Name wird bereits verwendet, bitte einen anderen auswählen." ) logging.info("Board name is valid.") # Render both templates enter_string_raw = str(enterMessageTemplateBlob.read()) show_string_raw = str(showMessageTemplateBlob.read()) for x in ["\\n", "\\r", "b'"]: enter_string_raw = enter_string_raw.replace(x, "") show_string_raw = show_string_raw.replace(x, "") enter_template = Template(enter_string_raw) show_template = Template(show_string_raw) rendered_enter_template = enter_template.render( board_id=board_id, iframe_page= f"https://adlsmessageboard.z1.web.core.windows.net/live_webpages/{board_id}/show_messages.html" ) rendered_show_template = show_template.render(board_id=board_id, content_items=[]) logging.info('Templates were processed.') #Write finished webpages to BLOB blob_client_enter = BlobClient( account_url="https://adlsmessageboard.blob.core.windows.net", container_name="$web", blob_name=f"live_webpages/{board_id}/enter_message.html", credential=os.environ["blob_key"]) blob_client_show = BlobClient( account_url="https://adlsmessageboard.blob.core.windows.net", container_name="$web", blob_name=f"live_webpages/{board_id}/show_messages.html", credential=os.environ["blob_key"]) logging.info('BLOB client was created.') blob_client_enter.upload_blob(rendered_enter_template, blob_type="BlockBlob", overwrite=True, content_settings=ContentSettings( content_type="text/html; charset=utf-8")) blob_client_show.upload_blob(rendered_show_template, blob_type="BlockBlob", overwrite=True, content_settings=ContentSettings( content_type="text/html; charset=utf-8")) enter_page_url = f"https://adlsmessageboard.z1.web.core.windows.net/live_webpages/{board_id}/enter_message.html" show_page_url = f"https://adlsmessageboard.z1.web.core.windows.net/live_webpages/{board_id}/show_messages.html" response_string = f"Die Websiten wurden erstellt!\n Eingabe-Website: {enter_page_url}\n Anzeige-Website: {show_page_url}" return func.HttpResponse(response_string)
blob_client_train = BlobClient(account_url="REPLACE", container_name=container_name, blob_name=local_file_name_train, credential="REPLACE") blob_client_forecast = BlobClient(account_url="REPLACE", container_name=container_name, blob_name=local_file_name_forecast, credential="REPLACE") blob_client_test = BlobClient(account_url="REPLACE", container_name=container_name, blob_name=local_file_name_test, credential="REPLACE") # Upload the created file with open(upload_file_path_train, "rb") as data: blob_client_train.upload_blob(data, overwrite=True) print("\nUploading to Azure Storage as blob:\n\t" + local_file_name_train) # blob_client_meta.upload_blob(meta_data, overwrite = True) # print("\nUploading to Azure Storage as blob:\n\t" + local_file_name_meta) with open(upload_file_path_forecast, "rb") as data: blob_client_forecast.upload_blob(data, overwrite=True) print("\nUploading to Azure Storage as blob:\n\t" + local_file_name_forecast) with open(upload_file_path_test, "rb") as data: blob_client_test.upload_blob(data, overwrite=True) print("\nUploading to Azure Storage as blob:\n\t" + local_file_name_test) # Add section for deletion here.
def main(event: func.EventGridEvent): event_data = event.get_json() symbol = event_data.get('symbol') statement = event_data.get('statement') logging.info(f'Statement: {statement}\nSymbol: {symbol}') # Define URL parameters for request to Alpha Vantage API params = {'apikey': API_KEY, 'function': statement, 'symbol': symbol} response = r.get(url=BASE_URL, params=params) try: data = response.json() except json.JSONDecodeError as e: logging.error(f'An error occurred while parsing response data.\n{e}') raise e else: a = data.get('annualReports', []) # Annual data q = data.get('quarterlyReports', []) # Get quarterly data # Upload annual data try: if len(a) > 0: target_blob_annual = BlobClient( account_url=DL_ACCOUNT_URL, container_name='alpha-vantage', blob_name= f'financial_statements/{statement}/annual/{symbol}.json', credential=DL_KEY) target_blob_annual.upload_blob(json.dumps(a), overwrite=True) logging.info('Succesfully uploaded annual data.') else: logging.info('No annual data present') except Exception as e: logging.error( f'An error occurred while uploading data to Data Lake.\n{e}') raise e # Upload quarterly data try: # Only upload when data is present if len(q) > 0: target_blob_quarterly = BlobClient( account_url=DL_ACCOUNT_URL, container_name='alpha-vantage', blob_name= f'financial_statements/{statement}/quarterly/{symbol}.json', credential=DL_KEY) target_blob_quarterly.upload_blob(json.dumps(q), overwrite=True) logging.info('Succesfully uploaded quarterly data.') else: logging.info('No quarterly data present.') except Exception as e: logging.error( f'An error occurred while uploading data to Data Lake.\n{e}') raise e
def main(req: func.HttpRequest, showMessageTemplateBlob: func.InputStream) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') board_id = req.params["board_id"] # Check if board_id already exists connection_str = os.environ["blobConnectionString"] blob_service_client = BlobServiceClient.from_connection_string( connection_str) container_name = "$web" container_client = blob_service_client.get_container_client(container_name) blob_list = container_client.list_blobs(name_starts_with="live_webpages/") existing_boards_list = list( set([b["name"].split("/")[1] for b in blob_list])) if board_id not in existing_boards_list: return func.HttpResponse("Dieses Board existiert nicht.") logging.info("Board name is valid.") # Read all contents for this board from storage table_service = TableService(account_name="adlsmessageboard", account_key=os.environ["blob_key"]) min_datetime_for_messages = ( datetime.datetime.utcnow() - datetime.timedelta(hours=24)).strftime("%Y-%m-%dT%H:%M:%SZ") messages_today = table_service.query_entities( "messages", filter= f"PartitionKey eq '{board_id}' and Timestamp gt datetime'{min_datetime_for_messages}'", select="sender_name, timestamp, message_text") message_list = [m for m in messages_today] for m in message_list: m.pop("etag") message_texts = [d["message_text"] for d in message_list] message_senders = [d["sender_name"] for d in message_list] message_timestamps = [d["timestamp"] for d in message_list] message_list_unique = [] distinct_message_texts = list(set(message_texts)) for t in distinct_message_texts: messages_this_sender = [ d for d in message_list if d["message_text"] == t ] text_sender = max([d["sender_name"] for d in messages_this_sender]) text_timestamp = max([d["timestamp"] for d in messages_this_sender]) message_list_unique.append({ "message_text": t, "sender_name": text_sender, "timestamp": text_timestamp }) message_list = sorted(message_list_unique, key=itemgetter("timestamp")) #Read images from storage connection_str = os.environ["blobConnectionString"] blob_service_client = BlobServiceClient.from_connection_string( connection_str) container_name = "$web" container_client = blob_service_client.get_container_client(container_name) blob_list = container_client.list_blobs( name_starts_with=f"images/{board_id}/") existing_images = sorted([b["name"] for b in blob_list], reverse=True) num_images = min(len(existing_images), 3) images_to_show = existing_images[0:num_images + 1] images_full_paths = [ f"https://adlsmessageboard.z1.web.core.windows.net/" + f for f in images_to_show ] image_items = [{"image_path": p} for p in images_full_paths] # Render template show_string_raw = str(showMessageTemplateBlob.read()) for x in ["\\n", "\\r", "b'"]: show_string_raw = show_string_raw.replace(x, "") show_template = Template(show_string_raw) rendered_show_template = show_template.render(board_id=board_id, content_items=message_list, image_items=image_items) logging.info('Template was processed.') #Write finished webpage to BLOB blob_client_show = BlobClient( account_url="https://adlsmessageboard.blob.core.windows.net", container_name="$web", blob_name=f"live_webpages/{board_id}/show_messages.html", credential=os.environ["blob_key"]) logging.info('BLOB client was created.') blob_client_show.upload_blob(rendered_show_template, blob_type="BlockBlob", overwrite=True, content_settings=ContentSettings( content_type="text/html; charset=utf-8")) return func.HttpResponse("Board wurde geupdatet.")
def uploadResults(funcData: dict, resultsBlobClient: BlobClient, results_obj: dict): try: results_string = json.dumps(results_obj, default=default_json_encoder) resultsBlobClient.upload_blob(results_string, overwrite=True) except Exception as _e: raise RuntimeError('Exiting due to unable to upload results file: {:s}. Error message: {:s}'.format(json.dumps(funcData), str(_e))) from _e