def run_sample(): try: # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService( account_name='adladandiag', account_key= 'PLkDD//pOQDCbJ/EzzAzfszFrLuj2RzpFkssg95IHIgmw1aY38/y0u81q1Ux6DJcp9t6XHHNnAy3WBcutIaTUg==' ) # Create a container called 'netmon-capture-logs'. container_name = 'netmon-capture-logs' # Set the permission so the blobs are public. block_blob_service.set_container_acl( container_name, public_access=PublicAccess.Container) # Create a file in Documents to test the upload and download. local_path = os.path.expanduser("C:\Network Traces\*") local_file_name = "test.cap" full_path_to_file = os.path.join(local_path, local_file_name) print("Temp file = " + full_path_to_file) print("\nUploading to Blob storage as blob" + local_file_name) # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file) except Exception as e: print(e)
def upload_to_azure_container(storage_file_name, local_path, storage_file_path): """Upload data to a storage account. Args: storage_file_name (String): The container to upload file to local_path (String): The full local file system path of the file storage_file_path (String): The file path to upload to within container Returns: (Boolean): True if file was uploaded """ uploaded = True try: account_key = str(os.environ.get('AZURE_ACCOUNT_KEY')) storage_account = str(os.environ.get('AZURE_STORAGE_ACCOUNT')) # Create the BlockBlockService that is used to call the # Blob service for the storage account. block_blob_service = BlockBlobService( account_name=storage_account, account_key=account_key) # Upload the created file, use local_file_name for the blob name. block_blob_service.create_blob_from_path( storage_file_name, storage_file_path, local_path) print(f'uploaded {storage_file_name} to {storage_account}') # pylint: disable=broad-except except Exception as error: print(error) uploaded = False return uploaded
def upload_blob(filename, filepath): if os.getenv("AZURE_STORAGE_ACCOUNT") and os.getenv("AZURE_STORAGE_KEY"): # Instantiate blob service block_blob_service = BlockBlobService(account_name=os.getenv("AZURE_STORAGE_ACCOUNT"), account_key=os.getenv("AZURE_STORAGE_KEY")) blob_name = filename # Upload blob try: block_blob_service.create_blob_from_path( os.getenv("AZURE_STORAGE_CONTAINER_NAME"), blob_name, filepath, content_settings=ContentSettings(content_type='image/jpg')) print 'Uploaded the photo: {}'.format(blob_name) print 'Check Storage Explorer under: {} > {} > {}'.format(os.getenv("AZURE_STORAGE_ACCOUNT"), os.getenv("AZURE_STORAGE_CONTAINER_NAME"), blob_name) print 'It is accessible at: https://{}.blob.core.windows.net/{}/{}'.format(os.getenv("AZURE_STORAGE_ACCOUNT"), os.getenv("AZURE_STORAGE_CONTAINER_NAME"), blob_name) except: print 'Unable to upload: {}'.format(blob_name) else: print 'Set your AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY in the .env file.'
def main(req: func.HttpRequest) -> func.HttpResponse: acc_name = req.params.get('account_name') acc_key = req.params.get('account_key') src_container_name = req.params.get('src_container_name') dst_container_name = req.params.get('dst_container_name') blob_name = req.params.get('blob_name') block_blob_service = BlockBlobService(account_name=acc_name, account_key=acc_key) download_path = "/tmp/" + blob_name start = time() block_blob_service.get_blob_to_path(src_container_name, blob_name, download_path) download_time = time() - start logging.info("Download time : " + str(download_time)) start = time() upload_path = download_path block_blob_service.create_blob_from_path(dst_container_name, upload_path.split("/")[2], upload_path) upload_time = time() - start logging.info("Upload time : " + str(upload_time)) result = "download_time : " + str( download_time) + "(s) upload_time : " + str(upload_time) + "(s)" return func.HttpResponse(result)
def upload_cmd_tree(): blob_file_name = 'extensionCommandTree.json' downloaded_file_name = 'extCmdTreeDownloaded.json' file_path = os.path.expanduser(os.path.join('~', '.azure', file_name)) client = BlockBlobService(account_name=STORAGE_ACCOUNT, account_key=STORAGE_ACCOUNT_KEY) client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=blob_file_name, file_path=file_path) url = client.make_blob_url(container_name=STORAGE_CONTAINER, blob_name=blob_file_name) download_file_path = os.path.expanduser( os.path.join('~', '.azure', downloaded_file_name)) download_file(url, download_file_path) if filecmp.cmp(file_path, download_file_path): print( "extensionCommandTree.json uploaded successfully. URL: {}".format( url)) else: raise Exception( "Failed to update extensionCommandTree.json in the storage account" )
def _upload_file_to_azure_storage_account(self, client: BlockBlobService, source: str, destination: str, container: str = None): """Upload the file to the specified Azure Storage Account. Assumption is that any cloud environment has access to a shared repository of artifacts. Args: client: Azure Storage Account client destination: Name of the file container: Name of the container the file should be uploaded to """ if not container: container = self.config["azure"]["common"][ "artifacts_shared_storage_account_container_name"] logger.info(f"""uploading artifact from | from ${source} | to ${destination} | in container {container}""") client.create_blob_from_path(container_name=container, blob_name=destination, file_path=source)
def upload_file_to_container(block_blob_client: azureblob.BlockBlobService, container_name: str, file_path: str, folder: str = None) -> batchmodels.ResourceFile: """ Uploads a local file to an Azure Blob storage container. :param block_blob_client: A blob service client. :param container_name: The name of the Azure Blob storage container. :param file_path: The local path to the file. :param folder: The folder on container to store the file, default None. :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ if folder: blob_name = f"{folder}/{os.path.basename(file_path)}" else: blob_name = os.path.basename(file_path) print('Uploading file {} to container [{}]...'.format( file_path, container_name)) block_blob_client.create_blob_from_path(container_name, blob_name, file_path) # Obtain the SAS token for the container. sas_token = get_container_sas_token(block_blob_client, container_name, azureblob.BlobPermissions.READ) sas_url = block_blob_client.make_blob_url(container_name, blob_name, sas_token=sas_token) return batchmodels.ResourceFile(file_path=blob_name, http_url=sas_url)
def upload_file(full_path_to_file): try: acc_name = 'accname' # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService(account_name=acc_name, account_key='acckey') # Create a container called 'quickstartblobs'. block_blob_service.create_container(container_name) # Set the permission so the blobs are public. block_blob_service.set_container_acl( container_name, public_access=PublicAccess.Container) local_file_name = "Bill.jpg" print("Temp file = " + full_path_to_file) print("\nUploading to Blob storage as blob" + local_file_name) # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file) # List the blobs in the container print("\nList blobs in the container") generator = block_blob_service.list_blobs(container_name) for blob in generator: print("\t Blob name: " + blob.name) return "https://" + acc_name + ".blob.core.windows.net/" + container_name + "/" + local_file_name except Exception as e: print(e)
def push2AzureAsBlobs(self): ''' push the files to azure ''' service = BlockBlobService(account_name=self.account_name, account_key=self.account_key) permission = ContainerPermissions(read=True, write=True) sas = service.generate_container_shared_access_signature( container_name=self.container_name, permission=permission, protocol='https', start=datetime.now(), expiry=datetime.now() + timedelta(days=1)) service = BlockBlobService(account_name=self.account_name, sas_token=sas) logfiles = self.getfilelist() for insfile in logfiles: print(insfile) basename = os.path.basename(insfile) if service.exists(container_name=self.container_name, blob_name=basename): print('File ' + basename + ' has been uploaded before.') else: service.create_blob_from_path( container_name=self.container_name, blob_name=basename, file_path=insfile, content_settings=ContentSettings( content_type=mimetypes.guess_type(basename)[0]), validate_content=False)
def upload_video(self, account_name='', sas_token='', container='', blob='', path=''): """Upload video to the provided account.""" block_blob_service = None if account_name and sas_token and container and blob: block_blob_service = BlockBlobService(account_name=account_name, sas_token=sas_token) containers = block_blob_service.list_containers() print("Available containers:") for container_entry in containers: print(container_entry.name) container_matches = any(x for x in containers if x.name == container) if not container_matches: block_blob_service.create_container(container) else: print('ERROR: No account credentials for Azure Storage specified.') block_blob_service.create_blob_from_path( container, blob, path, content_settings=ContentSettings(content_type='video/mp4'))
def run_azure(): try: # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService(connection_string='DefaultEndpointsProtocol=https;AccountName=flatscontainer;AccountKey=kx/bb+HLnwGFXwvT194g3Gl5EY8kM2OlG/Wb8El8b4yd/rjlcpqHHVjSJd0XG53DOxH4qiHszcU+sxogA52suA==;EndpointSuffix=core.windows.net', account_key='kx/bb+HLnwGFXwvT194g3Gl5EY8kM2OlG/Wb8El8b4yd/rjlcpqHHVjSJd0XG53DOxH4qiHszcU+sxogA52suA==') # Create a container. container_name ='datafromscraper' block_blob_service.create_container(container_name) # Set the permission so the blobs are public. block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container) # Create a file in Documents to test the upload and download. local_path=os.path.expanduser("..\dane") # <----------- ścieżka do pliku local_file_name ="flats_final.csv" # <---------------- nazwa pliku full_path_to_file =os.path.join(local_path, local_file_name) #print("Temp file = " + full_path_to_file) print("\nUploading to Blob storage as blob " + local_file_name) # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file) # List the blobs in the container print("\nList blobs in the container") generator = block_blob_service.list_blobs(container_name) for blob in generator: print("\t Blob name: " + blob.name) except Exception as e: print(e)
class AzureHelper: __instance = None __block_blob_service = None @staticmethod def getInstance(): if AzureHelper.__instance == None: AzureHelper() return AzureHelper.__instance def __init__(self): if AzureHelper.__instance != None: raise Exception("AzureHelper class is a singleton!") else: AzureHelper.__instance = self self.__block_blob_service = BlockBlobService( connection_string=Config.AZURE_CONNECTION_STRING) def pushToAzure(self, object_name): # push to azure if os.path.isfile("output/{}.csv".format(object_name)): try: self.__block_blob_service.create_blob_from_path( Config.AZURE_CONTAINER_NAME, Config.AZURE_FOLDER_NAME + "/{}.csv".format(object_name), "output/{}.csv".format(object_name)) print("^^^^^^ {} object pushed to Azure Storage Blob".format( object_name)) except Exception as inst: print(inst) IOHelper.appendToLog("azure_error.log", "\n\n{}".format(inst))
def run_sample(): try: # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService( account_name='pavic', account_key= 'lPdOTarDD4n3xuwz5EHMNOK3q/SNpnZ/1FKYKHNJSL4V4ZPt1LlvtjTZlpCfnghYFYQi20nTpBx6GVRPKfghoQ==' ) # Create a container called 'quickstartblobs'. container_name = 'aicamvideoclips' block_blob_service.create_container(container_name) #block_blob_service.cre # Set the permission so the blobs are public. block_blob_service.set_container_acl( container_name, public_access=PublicAccess.Container) # Create a file in Documents to test the upload and download. local_path = os.path.abspath(os.path.curdir) local_file_name = "Capture20190404_104457.mp4" # input("Enter file name to upload : ") full_path_to_file = os.path.join(local_path, local_file_name) # Write text to the file. #file = open(full_path_to_file, 'w') #file.write("Hello, World!") #file.close() print("Temp file = " + full_path_to_file) print("\nUploading to Blob storage as blob" + local_file_name) # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_path( container_name, "201904\\10\\" + local_file_name, full_path_to_file) # List the blobs in the container #print("\nList blobs in the container") #generator = block_blob_service.list_blobs(container_name) #for blob in generator: # print("\t Blob name: " + blob.name) # Download the blob(s). # Add '_DOWNLOADED' as prefix to '.txt' so you can see both files in Documents. #full_path_to_file2 = os.path.join(local_path, str.replace(local_file_name ,'.mp4', '_DOWNLOADED.mp4')) #print("\nDownloading blob to " + "201904\\10\\" +full_path_to_file2) #block_blob_service.get_blob_to_path(container_name, "201904\\10\\" +local_file_name, full_path_to_file2) sys.stdout.write( "Sample finished running. When you hit <any key>, the sample will be deleted and the sample " "application will exit.") sys.stdout.flush() input() # Clean up resources. This includes the container and the temp files #block_blob_service.delete_container(container_name) #os.remove(full_path_to_file) #os.remove(full_path_to_file2) except Exception as e: print(e)
def upload_file(self, file_name, object_name=None, **kwargs): if not self.check_if_sto_acc_exist(): logging.info( "I will create storage account {} in rg {} for you".format( self.sto_account, self.rg)) self.create_sto_account() if not self.test_if_obj_exist(object_name=self.sto_cont_name): logging.info("I will create {}:{}:{} for you".format( self.rg, self.sto_account, self.sto_cont_name)) self.create_sto_container() # if blob object name was not specified, use file_name if object_name is None: object_name = file_name sto_key = self.get_sto_account_key() blob = BlockBlobService(self.sto_account, sto_key) try: blob.create_blob_from_path(self.sto_cont_name, object_name, file_name) logging.info("File {} successfully uploaded to {}:{}:{}".format( file_name, self.sto_account, self.sto_cont_name, object_name)) except: logging.error("Cannot upload {} to {}:{}:{}".format( file_name, self.sto_account, self.sto_cont_name, object_name)) return False return True
def index(request): context = RequestContext(request) if request.method == 'POST': formobj = fileform(request.POST,request.FILES) if formobj.is_valid(): modelobj = formobj.save(commit=False) modelobj.image=request.FILES['image'] modelobj.save() mp3= request.FILES['image'] block_blob_service = BlockBlobService(account_name=myaccount, account_key=mykey) block_blob_service.create_container('harshultest2', public_access=PublicAccess.Container) block_blob_service.set_container_acl('harshultest2', public_access=PublicAccess.Container) with open('t.png','wb+') as formfile: for chunk in mp3.chunks(): formfile.write(chunk) block_blob_service.create_blob_from_path( 'harshultest2', str(request.FILES['image']), 't.png', content_settings=ContentSettings(content_type='image') ) return HttpResponseRedirect(reverse('list')) else: formobj = fileform() return render_to_response('app/myform.html', {'formobj': formobj},context)
def save_jsonfile(self, account, container_name, folder_name, local_file_name): try: block_blob_service = BlockBlobService(account.account_name, account.account_key) # Set the permission so the blobs are public. block_blob_service.set_container_acl( container_name, public_access=PublicAccess.Container) # Create a file in Documents to test the upload and download. full_path_to_file = os.path.join(folder_name, local_file_name) #Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file) # List the blobs in the container print("\nList blobs in the container") generator = block_blob_service.list_blobs(container_name) for blob in generator: print("\t Blob name: " + blob.name) sys.stdout.write("Don't forget to delete resources " "application will exit.") sys.stdout.flush() except Exception as e: print('Error occurred in create container.', e)
def process(): data = request.get_json() url = data['url'] pattern = re.compile(r'(?<=net\/).*?(?=\/)') match = pattern.findall(url) container_name = match[0] reg = r"(?<=" + container_name + "\/).*" pattern = re.compile(reg) match = pattern.findall(url) file_name1 = match[0] block_blob_service = BlockBlobService( account_name='dsconvreport', account_key= '2wIt3xVY2HR5mXfl2489ctyE1CIewgwA0am+jE85HkOfOBKc7Af0KHHb2YS9Z466T+v9KClZXYeht21M3oXFYw==' ) full_path_to_file = file_name1[:-5] + "_downloaded.xlsx" block_blob_service.get_blob_to_path(container_name, file_name1, full_path_to_file) df = pd.read_excel(full_path_to_file) val = df.values.tolist() items = [] supp_val = [] ranks = [] for i in val: if i[0] not in items: items.append(i[0]) for j in items: for i in val: if i[0] == j: supp_val.append(i[2]) ranks.extend(rd(supp_val, method='dense')) supp_val = [] df['supplier_rank'] = ranks files = 'Test_Supplierdata_kundan.xlsx' df.to_excel(files) block_blob_service.create_blob_from_path(container_name, 'Test_Supplierdata_kundan.xlsx', 'Test_Supplierdata_kundan.xlsx') op_url = "https://dsconvreport.blob.core.windows.net/" + container_name + "/" + files results[ 'result'] = "Congratulations, Your File has been Uploaded Successfully" results['uploaded_url'] = op_url return jsonify({ 'output_url': op_url, 'message': 'Your file has been downloaded and uploaded' })
def pagehandler(url, pageresponse, soup): """Function to be customized for processing of a single page. pageurl = URL of this page pageresponse = page content; response object from requests module soup = Beautiful Soup object created from pageresponse Return value = whether or not this page's links should be crawled. """ print("Crawling:" + url + " ({0} bytes)".format(len(pageresponse.text))) #url="https://www.bellevue.edu/student-support/career-services/pdfs/resume-samples.pdf" #get the page content and write in a file # Get the filename from the URL path try: # Create the BlockBlockService that is used to call the Blob service for the storage account # Comment below lines if you're not using BLOB block_blob_service = BlockBlobService(account_name='<ACCOUNT_NAME>', account_key='<ACCOUNT_KEY>') # Create a container container_name = '<CONTAINER_NAME>' # Set the permission so the blobs are public. block_blob_service.set_container_acl( container_name, public_access=PublicAccess.Container) filename = url[url.rfind("/") + 1:] filename_small = filename.replace(".", "_small.") #print("Filename " + filename) ## The following code takes the body of the content between the <div> tags of a particular id ## This is Optional. If you do not want to use to <DIV> comment the DIV related code div = soup.find("div", {"id": "<DIV ID>"}) div_text = div.get_text() ## Additional parsing if needed parsed_url = url.replace("<SOURCE STR>", "").replace("<CURRENT VALUE>", "<NEW VALUE>") print(div_text) if (div == None): print("None found") else: f = open("<LOCAL FOLDER NAME>/" + parsed_url, "a+") # write the content to file created #f.write(pageresponse.text) #r = requests.get(url, stream=True) #r = div #with open("extracted_data/" + filename, "wb") as f: #for chunk in r.iter_content(chunk_size=1024): #if chunk: f.write(div_text) block_blob_service.create_blob_from_path( container_name, parsed_url, '<LOCAL PATH>' + parsed_url) f.close() except: print("HTML Not processed:" + url) # wordcount(soup) # display unique word counts return True
def run_sample(localFileName, container): try: # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService( account_name='projectimage', account_key= 'JRchgJJAHjSzDMHMe73/9p65tTQTY7R9v/flqDfZagSTj00JiPXlAqi44B3P4Dkr3htQL3Eq2DAG81DXS7GdTw==' ) # Create a container called 'quickstartblobs'. container_name = container # 저장소 이름을 대문자 사용하지말자 # 대문자는 쓸 수 없는 문자라고 에러를 뱉는다. block_blob_service.create_container(container_name) # Set the permission so the blobsd are public. block_blob_service.set_container_acl( container_name, public_access=PublicAccess.Container) # # Create a file in Documents to test the upload and download. local_path = os.path.expanduser("~/BlobStorage") local_file_name = localFileName storage_url = METADATA.storage_url blob_url = storage_url + local_file_name # 이미지 분석에 사용 full_path_to_file = os.path.join(local_path, local_file_name) block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file) result_image_analyze = use_describe_image_api(blob_url) result_tran = translate_ko(result_image_analyze) return result_tran except Exception as e: print(e)
def upload_blob(self, filename, filepath): blob_service_client = BlockBlobService(account_name=self.container_acc, account_key=self.container_key) blob_service_client.set_container_acl( self.container_name, public_access=PublicAccess.Container) blob_service_client.create_blob_from_path(self.container_name, filename, filepath)
def take_picture(): print "take picture method called" # Turn on LED & Lamp GPIO.output(config.GPIO_PIN_ADDRESS, GPIO.HIGH) GPIO.output(config.LAMP_ADDRESS, GPIO.HIGH) # Take Picture camera = picamera.PiCamera() pic_id = "/home/pi/iot-hub-python-raspberrypi-client-app/imgs/" + str( uuid.uuid4()) + '.jpg' camera.capture(pic_id) camera.close() #turn camera off # Upload picture block_blob_service = BlockBlobService(account_name='yourBlobStorage', account_key='your-account-key') block_blob_service.create_blob_from_path( 'mycontainer', str(pic_id), str(pic_id), content_settings=ContentSettings(content_type='image/jpeg')) print "picture uploaded" #Turn off LED & Lamp GPIO.output(config.GPIO_PIN_ADDRESS, GPIO.LOW) GPIO.output(config.LAMP_ADDRESS, GPIO.LOW)
def save_image(image, blob_name): with open(blob_name, 'wb') as f: f.write(image) blobService = BlockBlobService(account_name=account_name, account_key=account_key) blobService.create_blob_from_path(container_name, blob_name, blob_name) os.remove(blob_name)
def upload_az(az_account_name, az_account_key, az_container, filename): print("[COMAE] Uploading to Azure container " + az_container) blobpath = os.path.basename(filename) block_blob_service = BlockBlobService(account_name=az_account_name, account_key=az_account_key) block_blob_service.create_container(az_container) block_blob_service.create_blob_from_path(az_container, blobpath, filename)
class BlobUploader(object): def __init__(self, blob_container=None, make_container_public=False): """ Class to handle uploading to an azure blob connection. :param make_container_public: True iff you are okay with public read access to your data. Useful for teaching a course :return: """ self.blob_container = blob_container or BLOB_CONTAINER self.blob_service = BlockBlobService(account_name=BLOB_ACCOUNTNAME, account_key=BLOB_ACCOUNTKEY) # if make_container_public: # self.blob_service.create_container(BLOB_CONTAINER, public_access=PublicAccess) # else: # self.blob_service.create_container(BLOB_CONTAINER) def put_json_file(self, file_obj, filename): """ Put a file into azure blob store. Allows user to specify format. For example, once could use: <prefix>/YYYYMMDD.json """ file_obj.seek(0) self.blob_service.create_blob_from_path( self.blob_container, filename, file_obj.name, content_settings=ContentSettings(content_type="text/json") )
def main(): img, filename = get_img() ############## FOR TESTING ######## #filename = "img.jpg" #img = plt.imread("img.jpg") ################################## json_res = send_api_helmet(filename) face_detected = send_face_recognition(filename) print_face_img(face_detected, img) tab = read_answer_api_helmet(json_res, img, re.split("/", filename)[-1]) date = datetime.today().strftime(("%Y-%m-%d")) csv_name = str(date) + ".csv" rows = create_row_csv(face_detected, tab) if len(rows) == 0: print("SAS closed. Unknow face detected") else: res = 0 for row in rows: print(row[1] + " was detected on the photo with " + row[3]) if row[3] == "no helmet": res = 1 if res == 0: print("SAS open.") else: print("SAS closed.") print(rows) add_data(csv_name, rows) block_blob_service = BlockBlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) block_blob_service.create_blob_from_path(CONTAINER_NAME_IMAGE, re.split("/", filename)[-1], filename) block_blob_service.create_blob_from_path(CONTAINER_NAME_CSV, csv_name, csv_name)
def uploadToAzure(files): block_blob_service = BlockBlobService(account_name=azureAccountName, account_key=azureAccountKey) totalFiles = str(len(files)) fileCount = 0 for file in files: fileCount += 1 print "Processing: " + file[1] + " (" + str( fileCount) + "/" + totalFiles + ")" filepath = os.path.join(file[0], file[1]) archivepath = file[0].replace(cameraPath, archivePath) if (filepath[-4:] == ".jpg") and (os.path.getsize(filepath) > 10485): try: containerName = file[0][20:28] block_blob_service.create_container(containerName) block_blob_service.create_blob_from_path( container_name=containerName, # Extract the date blob_name=file[1], # filename file_path=filepath, content_settings=ContentSettings( content_type=jpgContentType)) try: archiveFile(filepath, archivepath, file[1]) except OSError as e: print e.errno except: pass else: print "Incorrect extension or file size below threshold of 10MB. Archiving." archiveFile(filepath, archivepath, file[1])
def upload_icon(sas_url, file_path): # Break the SAS URL (scheme, netloc, path, params, query, fragment) = urlparse(sas_url) # Account is the first part of the netlocation upto the dot account_name = netloc[0:netloc.index('.')] # Container name is the path container_name = path.strip('/') # Create a block blob service blockblob_service = BlockBlobService(account_name=account_name, sas_token=query) # Get the file name of the icon file_name = os.path.basename(file_path) # Determine the content type and encoding for the file (content_type, content_encoding) = mimetypes.guess_type(file_name) content_settings = ContentSettings(content_type=content_type, content_encoding=content_encoding) # Upload the icon blockblob_service.create_blob_from_path(container_name=container_name, blob_name=file_name, file_path=file_path, content_settings=content_settings) # Append the icon name to the path to generate the download link path = path + '/' + file_name urlparts = (scheme, netloc, path, params, query, fragment) sas_download_url = urlunparse(urlparts) return sas_download_url
def uploadFileWithBlobSasUrl(self, file_name_full_path): retryRemaining = 3 while retryRemaining > 0: try: self.telemetryLogger.info("Cloud Environment: " + self.cloudEnv) sas_service = BlockBlobService( account_name=self.destination_storage_account, sas_token=self.destination_sas_token, endpoint_suffix=self.BlobUploadUrlSuffix()) self.telemetryLogger.info('Uploading to Blob starting.') start_time = datetime.now() sas_service.create_blob_from_path( self.destination_container_name, self.destination_blob_name, file_name_full_path) self.telemetryLogger.info( 'Uploading to Blob completed. Time take: ' + str((datetime.now() - start_time).total_seconds() * 1000) + ' ms') break except Exception as ex: retryRemaining -= 1 if retryRemaining <= 0: self.telemetryLogger.error( 'Encountered error during blob upload multiple times after exhausting all retry attempts' ) raise BlobUploadException(ex) exMessage = str(ex) self.telemetryLogger.warning( 'Encountered error during blob upload: ' + exMessage) self.telemetryLogger.warning('Retrying. ' + str(retryRemaining) + ' attempt(s) remaining.')
class RemoteStorage(): def __init__(self, user, key, container): self.bs = BlockBlobService(account_name=user, account_key=key) self.container = container def get_obj(self, blob_name): try: return self.bs.get_blob_properties(self.container, blob_name) except AzureMissingResourceHttpError: return None def download_blob(self, blob_name, localf): self.bs.get_blob_to_path(self.container, blob_name, localf) def del_blob(self, blob_name): try: self.bs.delete_blob(self.container, blob_name) except: pass def upload(self, blob_name, localf): self.bs.create_blob_from_path( self.container, blob_name, localf, content_settings=ContentSettings(content_type='text/xml')) def new_docs(self): res = [] for blob in self.bs.list_blobs(self.container): res.append(blob.name) return res
def write_file_to_blob(file_path, blob_name, container_name, bbs=None): if not bbs: bbs = BlockBlobService( account_name=config["storage_account_name"], account_key=config["storage_account_key"], ) bbs.create_blob_from_path(container_name, blob_name, file_path)
def download_and_reupload(self, twilio_filename): """ Download file from Twilio, upload to Azure, and return the Azure location and local file. """ blob_service = BlockBlobService(account_name=blob_accountname, account_key=blob_key) response = requests.get(twilio_filename) if response.status_code != 200: raise TwilioResponseError( "Couldn't find Twilio file {0}".format(twilio_filename)) suffix = "wav" short_file_name = "{0}.{1}".format(uuid.uuid4(), suffix) azure_path = "/recordings/" + short_file_name with TmpFileCleanup() as tmp_file_store: # local_filename = local_tmp_dir + "/" + short_file_name local_filename = os.path.join(local_tmp_dir, short_file_name) tmp_file_store.tmp_files.append(local_filename) f = open(local_filename, "wb") f.write(response.content) f.close() # Now upload the local file to azure blob_service.create_blob_from_path( blob_container, azure_path, local_filename, content_settings=ContentSettings(content_type='audio/x-wav')) return azure_path
def run_sample(): try: # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService( account_name='meetpythonstorage', account_key='abcabcabcabcabcabcabcabcabcabcabcabcabac') # Create a container called 'quickstartblobs'. container_name = 'quickstartblobs' block_blob_service.create_container(container_name) # Set the permission so the blobs are public. block_blob_service.set_container_acl( container_name, public_access=PublicAccess.Container) # Create a file in Documents to test the upload and download. local_path = os.path.abspath(os.path.curdir) local_file_name = input("Enter file name to upload : ") full_path_to_file = os.path.join(local_path, local_file_name) # Write text to the file. #file = open(full_path_to_file, 'w') #file.write("Hello, World!") #file.close() print("Temp file = " + full_path_to_file) print("\nUploading to Blob storage as blob" + local_file_name) # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file) # List the blobs in the container print("\nList blobs in the container") generator = block_blob_service.list_blobs(container_name) for blob in generator: print("\t Blob name: " + blob.name) # Download the blob(s). # Add '_DOWNLOADED' as prefix to '.txt' so you can see both files in Documents. full_path_to_file2 = os.path.join( local_path, str.replace(local_file_name, '.txt', '_DOWNLOADED.txt')) print("\nDownloading blob to " + full_path_to_file2) block_blob_service.get_blob_to_path(container_name, local_file_name, full_path_to_file2) sys.stdout.write( "Sample finished running. When you hit <any key>, the sample will be deleted and the sample " "application will exit.") sys.stdout.flush() input() # Clean up resources. This includes the container and the temp files block_blob_service.delete_container(container_name) os.remove(full_path_to_file) os.remove(full_path_to_file2) except Exception as e: print(e)
def test_get_put_blob(self): import config as config account_name = config.STORAGE_ACCOUNT_NAME account_key = config.STORAGE_ACCOUNT_KEY block_blob_service = BlockBlobService(account_name, account_key) block_blob_service.create_blob_from_path( 'cont2', 'sunset.png', 'sunset.png',) block_blob_service.get_blob_to_path('cont2', 'sunset.png', 'out-sunset.png')
class _BlobStorageFileHandler(object): def __init__(self, account_name=None, account_key=None, protocol='https', container='logs', zip_compression=False, max_connections=1, max_retries=5, retry_wait=1.0, is_emulated=False): self.service = BlockBlobService(account_name=account_name, account_key=account_key, is_emulated=is_emulated, protocol=protocol) self.container_created = False hostname = gethostname() self.meta = {'hostname': hostname.replace('_', '-'), 'process': os.getpid()} self.container = (container % self.meta).lower() self.meta['hostname'] = hostname self.zip_compression = zip_compression self.max_connections = max_connections self.max_retries = max_retries self.retry_wait = retry_wait def put_file_into_storage(self, dirName, fileName): """ Ship the outdated log file to the specified blob container. """ if not self.container_created: self.service.create_container(self.container) self.container_created = True fd, tmpfile_path = None, '' try: file_path = os.path.join(dirName, fileName) if self.zip_compression: suffix, content_type = '.zip', 'application/zip' fd, tmpfile_path = mkstemp(suffix=suffix) with os.fdopen(fd, 'wb') as f: with ZipFile(f, 'w', ZIP_DEFLATED) as z: z.write(file_path, arcname=fileName) file_path = tmpfile_path else: suffix, content_type = '', 'text/plain' self.service.create_blob_from_path(container_name=self.container, blob_name=fileName+suffix, file_path=file_path, content_settings=ContentSettings(content_type=content_type), max_connections=self.max_connections ) # max_retries and retry_wait no longer arguments in azure 0.33 finally: if self.zip_compression and fd: os.remove(tmpfile_path)
def upload_file(STORAGE_NAME, STORAGE_KEY, NEW_CONTAINER_NAME, file, path, extension, content_type): """create blob service, and upload files to container""" blob_service = BlockBlobService(account_name= STORAGE_NAME, account_key=STORAGE_KEY) try: blob_service.create_blob_from_path(NEW_CONTAINER_NAME, file, path, content_settings=ContentSettings(content_type= content_type+extension)) print("{} // BLOB upload status: successful".format(file)) except: print("{} // BLOB upload status: failed".format(file))
def run_sample(): try: # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService(account_name='meetpythonstorage', account_key='duOguiKnYb6ZEbJC6BftWqA2lcH67dWkmCSEJj+KxOTOHCNPeV7r4oO6feTw7gSSoFGKHryL4yqSVWlEkm6jWg==') # Create a container called 'quickstartblobs'. container_name ='quickstartblobs' block_blob_service.create_container(container_name) # Set the permission so the blobs are public. block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container) # Create a file in Documents to test the upload and download. local_path=os.path.abspath(os.path.curdir) local_file_name =input("Enter file name to upload : ") full_path_to_file =os.path.join(local_path, local_file_name) # Write text to the file. #file = open(full_path_to_file, 'w') #file.write("Hello, World!") #file.close() print("Temp file = " + full_path_to_file) print("\nUploading to Blob storage as blob" + local_file_name) # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file) # List the blobs in the container print("\nList blobs in the container") generator = block_blob_service.list_blobs(container_name) for blob in generator: print("\t Blob name: " + blob.name) # Download the blob(s). # Add '_DOWNLOADED' as prefix to '.txt' so you can see both files in Documents. full_path_to_file2 = os.path.join(local_path, str.replace(local_file_name ,'.txt', '_DOWNLOADED.txt')) print("\nDownloading blob to " + full_path_to_file2) block_blob_service.get_blob_to_path(container_name, local_file_name, full_path_to_file2) sys.stdout.write("Sample finished running. When you hit <any key>, the sample will be deleted and the sample " "application will exit.") sys.stdout.flush() input() # Clean up resources. This includes the container and the temp files block_blob_service.delete_container(container_name) os.remove(full_path_to_file) os.remove(full_path_to_file2) except Exception as e: print(e)
def spider_closed(self, spider): self.exporter.finish_exporting() file = self.files.pop(spider) filename = file.name newname = filename[:-5]+'-'+datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")+'.tsv' file.close() os.rename(filename, newname) if UPLOAD_TO_AZURE_STORAGE: block_blob_service = BlockBlobService(account_name=AZURE_ACCOUNT_NAME, account_key=AZURE_ACCOUNT_KEY) block_blob_service.create_blob_from_path(AZURE_CONTAINER, newname, newname, content_settings=ContentSettings(content_type='text/tab-separated-values') )
def _upload_package_blob(package_file, url): """Upload the location file to storage url provided by autostorage""" uri = urlsplit(url) # in uri path, it always start with '/', so container name is at second block pathParts = uri.path.split("/", 2) container_name = pathParts[1] blob_name = pathParts[2] # we need handle the scenario storage account not in public Azure hostParts = uri.netloc.split(".", 2) account_name = hostParts[0] # endpoint suffix needs to ignore the 'blob' part in the host name endpoint_suffix = hostParts[2] sas_service = BlockBlobService(account_name=account_name, sas_token=uri.query, endpoint_suffix=endpoint_suffix) sas_service.create_blob_from_path(container_name=container_name, blob_name=blob_name, file_path=package_file)
def publish(build, account, container, sas, **_) -> None: client = BlockBlobService(account_name=account, sas_token=sas) publishing_files = (p for p in glob.iglob(os.path.join(build , '**/*'), recursive=True)) for source in publishing_files: if os.path.isdir(source): continue blob_path = os.path.join(os.environ['TRAVIS_REPO_SLUG'], os.environ['TRAVIS_BRANCH'], os.environ['TRAVIS_BUILD_NUMBER'], os.path.relpath(source, build)) content_type, content_encoding = mimetypes.guess_type(os.path.basename(source)) content_settings = ContentSettings(content_type, content_encoding) logger.info(f'Uploading {blob_path} ...') client.create_blob_from_path(container_name=container, blob_name=blob_path, file_path=source, content_settings=content_settings)
def upload_file_to_container(block_blob_client: azureblob.BlockBlobService, container_name: str, file_path: str): """ Uploads a local file to an Azure Blob storage container. :param block_blob_client: A blob service client. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param container_name: The name of the Azure Blob storage container. :type container_name: str :param str file_path: The local path to the file. :type file_path: str """ blob_name = os.path.basename(file_path) logger.info( 'Uploading file [{}] to container [{}]...'.format( file_path, container_name)) block_blob_client.create_blob_from_path(container_name, blob_name, file_path)
def nightly(build: str, account: str, container: str, sas: str, **_) -> None: client = BlockBlobService(account_name=account, sas_token=sas) modules_list = [] for wheel_file in glob.iglob(os.path.join(build, 'build/*.whl')): package_name = os.path.basename(wheel_file).split('-', maxsplit=1)[0].replace('_', '-') sdist_file = next(glob.iglob(os.path.join(build, 'source', f'{package_name}*.tar.gz'))) content_type, content_encoding = mimetypes.guess_type(os.path.basename(wheel_file)) content_settings = ContentSettings(content_type, content_encoding) client.create_blob_from_path(container_name=container, blob_name=f'{package_name}/{os.path.basename(wheel_file)}', file_path=wheel_file, content_settings=content_settings) content_type, content_encoding = mimetypes.guess_type(os.path.basename(sdist_file)) content_settings = ContentSettings(content_type, content_encoding) client.create_blob_from_path(container_name=container, blob_name=f'{package_name}/{os.path.basename(sdist_file)}', file_path=sdist_file, content_settings=content_settings) package_blobs = (os.path.basename(b.name) for b in client.list_blobs(container, prefix=package_name + '/') if b.name != f"{package_name}/") client.create_blob_from_text(container_name=container, blob_name=f'{package_name}/', text=generate_package_list_in_html(f'Links for {package_name}', package_blobs), content_settings=ContentSettings('text/html')) modules_list.append(f"{package_name}/") client.create_blob_from_text(container_name=container, blob_name='index.html', text=generate_package_list_in_html('Simple Index', modules_list), content_settings=ContentSettings('text/html'))
from gpiozero import MotionSensor from picamera import PiCamera from datetime import datetime from azure.storage.blob import BlockBlobService from azure.storage.blob import ContentSettings camera = PiCamera() pir = MotionSensor(4) block_blob_service = BlockBlobService(account_name='YOURACCOUNT', account_key='YOURKEY') while True: pir.wait_for_motion() filename = "pircam-" + datetime.now().strftime("%Y-%m-%d_%H.%M.%S.jpg") camera.capture(filename) pir.wait_for_no_motion() block_blob_service.create_blob_from_path( 'YOURCONTAINER', filename, filename, content_settings=ContentSettings(content_type='image/jpeg'))
policy_name = "readandlistonly" # The Storage Account Name storage_account_name = "mystore" storage_account_key = "mykey" storage_container_name = "mycontainer" example_file_path = "..\\sampledata\\sample.log" policy_name = "mysaspolicy" # Create the blob service, using the name and key for your Azure Storage account blob_service = BlockBlobService(storage_account_name, storage_account_key) # Create the container, if it does not already exist blob_service.create_container(storage_container_name) # Upload an example file to the container blob_service.create_blob_from_path(storage_container_name, "sample.log", example_file_path) # Create a new policy that expires after a week access_policy = AccessPolicy( permission=ContainerPermissions.READ + ContainerPermissions.LIST, expiry=datetime.utcnow() + timedelta(weeks=1) ) # Get the existing identifiers (policies) for the container identifiers = blob_service.get_container_acl(storage_container_name) # And add the new one ot the list identifiers[policy_name] = access_policy # Set the container to the updated list of identifiers (policies) blob_service.set_container_acl(storage_container_name, identifiers)
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, bucket_name, prefix=None): prefix = "{}".format(prefix.lstrip("/") if prefix else "") super().__init__(prefix=prefix) self.account_name = account_name self.account_key = account_key self.container_name = bucket_name self.conn = BlockBlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized, %r", self.container_name) def get_metadata_for_key(self, key): key = self.format_key_for_backend(key, remove_slash_prefix=True, trailing_slash=False) results = self._list_blobs(key) if not results: raise FileNotFoundFromStorageError(key) return results[0]["metadata"] def _metadata_for_key(self, key): return self._list_blobs(key)[0]["metadata"] def list_path(self, key, trailing_slash=True): # pylint: disable=arguments-differ # Trailing slash needed when listing directories, without when listing individual files path = self.format_key_for_backend(key, remove_slash_prefix=True, trailing_slash=trailing_slash) return self._list_blobs(path) def _list_blobs(self, path): self.log.debug("Listing path %r", path) if path: items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata") else: # If you give Azure an empty path, it gives you an authentication error items = self.conn.list_blobs(self.container_name, delimiter="/", include="metadata") results = [] for item in items: if not isinstance(item, BlobPrefix): results.append({ "last_modified": item.properties.last_modified, # Azure Storage cannot handle '-' so we turn them into underscores and back again "metadata": dict((k.replace("_", "-"), v) for k, v in item.metadata.items()), "name": self.format_key_from_backend(item.name), "size": item.properties.content_length, }) return results def delete_key(self, key): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.log.debug("Deleting key: %r", key) try: return self.conn.delete_blob(self.container_name, key) except azure.common.AzureMissingResourceHttpError as ex: raise FileNotFoundFromStorageError(key) from ex def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=None): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to) try: self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to) except azure.common.AzureMissingResourceHttpError as ex: raise FileNotFoundFromStorageError(key) from ex if progress_callback: progress_callback(1, 1) return self._metadata_for_key(key) def get_contents_to_fileobj(self, key, fileobj_to_store_to, *, progress_callback=None): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.log.debug("Starting to fetch the contents of: %r", key) try: self.conn.get_blob_to_stream(self.container_name, key, fileobj_to_store_to) except azure.common.AzureMissingResourceHttpError as ex: raise FileNotFoundFromStorageError(key) from ex if progress_callback: progress_callback(1, 1) return self._metadata_for_key(key) def get_contents_to_string(self, key): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.log.debug("Starting to fetch the contents of: %r", key) try: blob = self.conn.get_blob_to_bytes(self.container_name, key) return blob.content, self._metadata_for_key(key) except azure.common.AzureMissingResourceHttpError as ex: raise FileNotFoundFromStorageError(key) from ex def store_file_from_memory(self, key, memstring, metadata=None): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.conn.create_blob_from_bytes(self.container_name, key, memstring, metadata=self.sanitize_metadata(metadata, replace_hyphen_with="_")) def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): key = self.format_key_for_backend(key, remove_slash_prefix=True) self.conn.create_blob_from_path(self.container_name, key, filepath, metadata=self.sanitize_metadata(metadata, replace_hyphen_with="_")) def get_or_create_container(self, container_name): start_time = time.monotonic() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.monotonic() - start_time) return container_name
class AzureStorage(plugins.IStoragePlugin): def configure(self, config): self.storage = BlockBlobService(account_name=config['account_name'], account_key=config['account_key']) self.container = config['container'] try: container = self.storage.get_container_properties(self.container) log.info("Configuring Azure blob storage %s/%s", self.storage.account_name, self.container) except AzureMissingResourceHttpError as e: log.warning("Container '%s' is missing in account '%s', trying to create new", self.container, self.storage.account_name) try: self.storage.create_container(self.container) self.storage.set_container_acl(self.container, public_access=PublicAccess.Container) except Exception as e: log.critical("Cannot create new container: %s", e) raise plugins.PluginInitException("Cannot create new container") except AzureHttpError as e: log.critical("Cannot access container '%s' in account '%s': %s", self.container, self.storage.account_name, e) raise plugins.PluginInitException("Cannot access container") except Exception as e: log.critical("Cannot access container '%s' in account '%s': %s", self.container, self.storage.account_name, e) raise plugins.PluginInitException("Cannot access container") def delete(self, key): log.info("Deleting file '%s' from %s/%s", key, self.storage.account_name, self.container) try: self.storage.delete_blob(self.container, key) except AzureMissingResourceHttpError: log.error("File '%s' was not found in %s/%s", key, self.storage.account_name, self.container) raise common.NotFound('File not found') except Exception as e: log.error("Cannot delete '%s' from %s/%s: %s", key, self.storage.account_name, self.container, e) raise common.FatalError(e) def put(self, key, filename=None, file=None): storage_key = key try: if filename: log.debug("Uploading %s to %s", filename, self.storage.make_blob_url(self.container, storage_key)) self.storage.create_blob_from_path(self.container, storage_key, filename, content_settings=ContentSettings(content_type='application/octet-stream')) elif file: old_pos = file.tell() file.seek(0) log.debug("Uploading from stream to %s", self.storage.make_blob_url(self.container, storage_key)) self.storage.create_blob_from_stream(self.container, storage_key, file, content_settings=ContentSettings(content_type='application/octet-stream')) file.seek(old_pos) except Exception as e: # TODO: more detailed error inspection log.critical("Error uploading to %s/%s: %s", self.storage.account_name, self.container, e) raise common.FatalError(e) return storage_key def get(self, key, stream): # current azure python sdk barely can work with non-seekable streams, # so we have to implement chunking by our own # TODO: proper ranging? RFC says server SHOULD return 406 once range is unsatisfiable, # but Azure is OK with end pos > blob length unless blob is not empty chunk_size = 4*1024*1024 chunk_start = 0 chunk_end = chunk_size - 1 while True: try: chunk = self.storage._get_blob(self.container, key, start_range=chunk_start, end_range=chunk_end) log.debug("Writing %s bytes from %s", len(chunk.content), chunk_start) stream.write(chunk.content) except IOError: # remote side closed connection return except AzureMissingResourceHttpError as e: raise common.NotFound(e) except (AzureHttpError, AzureException) as e: raise common.TemporaryError('Error while downloading {}: {}'.format(key, e)) blob_length = int(chunk.properties.content_range.split('/')[1]) chunk_start, chunk_end, blob_size = map(int, re.match(r'^bytes\s+(\d+)-(\d+)/(\d+)$', chunk.properties.content_range).groups()) if chunk_end == blob_size - 1: # no more data to stream break else: chunk_start = chunk_end + 1 chunk_end += chunk_size return 0
class AzureBlobWriter(BaseWriter): """ Writes items to azure blob containers. - account_name (str) Public acces name of the azure account. - account_key (str) Public acces key to the azure account. - container (str) Blob container name. """ supported_options = { 'account_name': {'type': six.string_types, 'env_fallback': 'EXPORTERS_AZUREWRITER_NAME'}, 'account_key': {'type': six.string_types, 'env_fallback': 'EXPORTERS_AZUREWRITER_KEY'}, 'container': {'type': six.string_types} } hash_algorithm = 'md5' VALID_CONTAINER_NAME_RE = r'[a-zA-Z0-9-]{3,63}' def __init__(self, *args, **kw): from azure.storage.blob import BlockBlobService super(AzureBlobWriter, self).__init__(*args, **kw) account_name = self.read_option('account_name') account_key = self.read_option('account_key') self.container = self.read_option('container') if '--' in self.container or not re.match(self.VALID_CONTAINER_NAME_RE, self.container): help_url = ('https://azure.microsoft.com/en-us/documentation' '/articles/storage-python-how-to-use-blob-storage/') warnings.warn("Container name %s doesn't conform with naming rules (see: %s)" % (self.container, help_url)) self.azure_service = BlockBlobService(account_name, account_key) self.azure_service.create_container(self.container) self.logger.info('AzureBlobWriter has been initiated.' 'Writing to container {}'.format(self.container)) self.set_metadata('files_counter', 0) self.set_metadata('blobs_written', []) def write(self, dump_path, group_key=None): self.logger.info('Start uploading {} to {}'.format(dump_path, self.container)) self._write_blob(dump_path) self.set_metadata('files_counter', self.get_metadata('files_counter') + 1) @retry_long def _write_blob(self, dump_path): blob_name = dump_path.split('/')[-1] self.azure_service.create_blob_from_path( self.read_option('container'), blob_name, dump_path, max_connections=5, ) self.logger.info('Saved {}'.format(blob_name)) self._update_metadata(dump_path, blob_name) def _update_metadata(self, dump_path, blob_name): buffer_info = self.write_buffer.metadata[dump_path] file_info = { 'blob_name': blob_name, 'size': buffer_info['size'], 'hash': b64encode(unhexlify(buffer_info['file_hash'])), 'number_of_records': buffer_info['number_of_records'] } self.get_metadata('blobs_written').append(file_info) def _check_write_consistency(self): from azure.common import AzureMissingResourceHttpError for blob_info in self.get_metadata('blobs_written'): try: blob = self.azure_service.get_blob_properties( self.read_option('container'), blob_info['blob_name']) blob_size = blob.properties.content_length blob_md5 = blob.properties.content_settings.content_md5 if str(blob_size) != str(blob_info['size']): raise InconsistentWriteState( 'File {} has unexpected size. (expected {} - got {})'.format( blob_info['blob_name'], blob_info['size'], blob_size ) ) if str(blob_md5) != str(blob_info['hash']): raise InconsistentWriteState( 'File {} has unexpected hash. (expected {} - got {})'.format( blob_info['blob_name'], blob_info['hash'], blob_md5 ) ) except AzureMissingResourceHttpError: raise InconsistentWriteState('Missing blob {}'.format(blob_info['blob_name'])) self.logger.info('Consistency check passed')
class BlobStorage(object): # void def __init__(self, wasbs_path, dryrun=False): parsed = urlparse(wasbs_path) if parsed.scheme not in ('wasbs', 'wasb'): raise InvalidBlobStorePath('Remote path is not supported! Expected format: `wasb[s]://container/blob-path`') self.dryrun = dryrun self.schema, self.container, self.blob_path = parsed.scheme, parsed.netloc, parsed.path if self.blob_path and self.blob_path[0] == u'/': self.blob_path = self.blob_path[1:] self.blob_path = self.blob_path or None self.pbar = None self.service = BlockBlobService( account_name=os.environ['AZURE_STORAGE_ACCOUNT'].strip(), account_key=os.environ['AZURE_STORAGE_ACCESS_KEY'].strip()) @property def url(self): return u'{}://{}'.format(self.schema, self.container) @property def path(self): return os.path.join(self.url, self.blob_path) # Blob def get_blob(self): for blob in self.list_blobs(): if blob.path == self.blob_path: return blob # genexp<list<Blob>> def list_blobs(self): marker = None while True: batch = self.service.list_blobs(self.container, prefix=self.blob_path, marker=marker) for blob in batch: yield Blob(self, blob) if not batch.next_marker: break marker = batch.next_marker # void def execute(self, executable_fn, message, end=None, **kwargs): # Print the original message print(message % kwargs, end=end) # If dryrun, write the message and exit if self.dryrun: print('IGNORE (--dryrun)') return try: executable_fn(**kwargs) print('OK') except Exception as e: print('FAIL\n{}'.format(e)) # void def remove_fn(self, path, url=None): self.service.delete_blob(self.container, path) # void def remove_blobs(self, prefix=False): if not self.blob_path: print(u'Have to specify the path of the blob.') sys.exit(1) if not prefix: return self.execute(self.remove_fn, 'Remove blob from `%(url)s` ... ', path=self.blob_path, url=self.path, end='') for blob in self.list_blobs(): self.execute(self.remove_fn, 'Remove blob from `%(url)s` ... ', path=blob.path, url=blob.url, end='') # void def upload_fn(self, blob_path, file_path, rel_file_path=None, url=None): self.service.create_blob_from_path(self.container, blob_path, file_path, \ max_connections=int(os.environ.get('AZURE_STORAGE_MAX_CONNECTIONS',1)), \ progress_callback=self.show_progress) self.pbar.finish() self.pbar = None # tuple<str,str> def get_upload_path_pair(self, file_path, common_prefix=None): is_directory_ending = self.blob_path and self.blob_path.endswith('/') is_container_path = self.blob_path is None blob_path = os.path.join(self.blob_path or u'', os.path.split(file_path)[-1]) \ if any([is_container_path, is_directory_ending]) and common_prefix is None \ else self.blob_path if common_prefix and blob_path: blob_path = os.path.join(blob_path, file_path.split(common_prefix)[-1].strip('/')) elif common_prefix and not blob_path: blob_path = file_path.split(common_prefix)[-1].strip('/') elif common_prefix == u'' and blob_path: blob_path = os.path.join(blob_path, file_path.strip('/')) elif common_prefix == u'' and not blob_path: blob_path = file_path.strip('/') return (file_path, blob_path) # genexp<tuple<str,str>> def get_upload_path_pairs(self, file_paths): if len(file_paths) == 1: yield self.get_upload_path_pair(file_paths[0]) return common_prefix = os.path.split(os.path.commonprefix(file_paths))[0] if self.blob_path and not self.blob_path.endswith('/'): self.blob_path += '/' for file_path in file_paths: yield self.get_upload_path_pair(file_path, common_prefix=common_prefix) # void def upload_blobs(self, file_paths): for file_path, blob_path in self.get_upload_path_pairs(file_paths): self.execute(self.upload_fn, 'Upload `%(rel_file_path)s` into `%(url)s`', \ file_path=file_path, rel_file_path=os.path.relpath(file_path), blob_path=blob_path, \ url=u'{}/{}'.format(self.url, blob_path)) # void def show_progress(self, current, total): def filesize(n,pow=0,b=1024,u='B',pre=['']+[p+'i'for p in'KMGTPEZY']): pow,n=min(int(log(max(n*b**pow,1),b)),len(pre)-1),n*b**pow return "%%.%if %%s%%s"%abs(pow%(-pow-1))%(n/b**float(pow),pre[pow],u) if self.pbar is None: self.pbar = ProgressBar(widgets=[ ' '*5, 'Size: {}'.format(filesize(total)), ' ', Percentage(), ' ', Bar(), ' ', ETA(), ' ', FileTransferSpeed(), ' '*5 ], maxval=total ).start() self.pbar.update(current) # void def download_fn(self, blob_path, file_path, **kwargs): self.service.get_blob_to_path(self.container, blob_path, file_path, \ max_connections=int(os.environ.get('AZURE_STORAGE_MAX_CONNECTIONS',1)), \ progress_callback=self.show_progress) self.pbar.finish() self.pbar = None # tuple<str,str> def get_download_path_pair(self, blob_path, file_path, common_prefix=None): file_path = os.path.join(file_path, os.path.split(blob_path)[-1]) \ if os.path.exists(file_path) and os.path.isdir(file_path) and common_prefix is None \ else file_path if common_prefix: file_path = os.path.join(file_path, blob_path.split(common_prefix)[-1].strip('/')) elif common_prefix == u'': file_path = os.path.join(file_path, blob_path.strip('/')) dir_path = os.path.split(file_path)[0] if dir_path and not os.path.exists(dir_path): os.makedirs(dir_path) return blob_path, file_path # genexp<tuple<str,str>> def get_download_path_pairs(self, file_path, prefix=False, skip_existing=False, sync=False): # Ignore if no blob path is defined. if not self.blob_path: raise BlobPathRequired(u'Blob path is required for `get` command.') # Single file download scenario. if not prefix: # Skip if the file is already existing. if skip_existing and os.path.exists(file_path): return # Only downloads the not existing or the updated files (based on file size). if sync and os.path.exists(file_path): blob = self.get_blob() if blob and get_fresher(blob, file_path) != blob: return # Return the caluclated path of the file. yield self.get_download_path_pair(self.blob_path, file_path) return # List the blobs with the given prefix in the ABS. blob_paths, blob_paths_dict = [], {} for blob in self.list_blobs(): blob_paths.append(blob.path) blob_paths_dict[blob.path] = blob # Determine the common prefix between the blobs. common_prefix = os.path.dirname(self.blob_path) \ if not self.blob_path.endswith('/') \ else self.blob_path resolved_file_paths = [] for blob_path in blob_paths: # Determine the input, output path pairs. bp, fp = self.get_download_path_pair(blob_path, file_path, common_prefix=common_prefix) # If any of the files want to write to the same file, raise an error. if fp in resolved_file_paths: raise DirectoryRequired('Can not use the same path (`{}`) for multiple blob!' \ .format(fp)) # Ignore the files that already exists. if skip_existing and os.path.exists(fp): continue # Only downloads the not existing or the updated files (based on file size). if sync and os.path.exists(fp) and get_fresher(blob_paths_dict[blob_path], fp) != blob_paths_dict[blob_path]: continue resolved_file_paths.append(fp) yield bp, fp # void def download_blobs(self, file_path, prefix=False, skip_existing=False, sync=False): # Iterates over the final input, output paths and download them. for blob_path, file_path in self.get_download_path_pairs(file_path, prefix=prefix, skip_existing=skip_existing, sync=sync): self.execute(self.download_fn, 'Download `%(url)s` into `%(rel_file_path)s`', \ blob_path=blob_path, file_path=file_path, rel_file_path=os.path.relpath(file_path), \ url=u'{}/{}'.format(self.url, blob_path))
return json_fpath if __name__ == "__main__": parser = argparse.ArgumentParser(description="Processes and stores data into hbase") parser.add_argument("--container",dest="container") parser.add_argument("--pattern",dest="pattern") args = parser.parse_args() container = args.container pattern = args.pattern print("Processing files from container : " + str(container)) if pattern is not None: blob_list = block_blob_service.list_blobs(container_name=container, prefix=pattern) else: blob_list = block_blob_service.list_blobs(container_name=container) for blob in blob_list: print("Processing blob : " + blob.name) blob_name = getfilename(blob.name) downloadedblob = "downloaded_" + blob_name block_blob_service.get_blob_to_path(container_name=container,blob_name=blob.name, file_path=downloadedblob, open_mode='w') if pattern is not None: json_outpath = processcsvfile(fname=downloadedblob,seperator="|",outfname=blob_name,outdir='jsonfiles/' + container + "/" + pattern) print("uploading blob" + json_outpath) block_blob_service.create_blob_from_path(container_name=container,blob_name=str(pattern+ 'json/' + blob_name + ".json"),file_path=json_outpath) else: json_outpath = processcsvfile(fname=downloadedblob,seperator="|",outfname=blob_name,outdir='jsonfiles/' + container + "/") print("uploading blob" + json_outpath) block_blob_service.create_blob_from_path(container_name=container,blob_name=str('json/' + blob_name + ".json"),file_path=json_outpath)
api_key = "56395e3fd1f33296c424c2c6e3ee008a" lat = i.latitude lng = i.longitude forecast = forecastio.load_forecast(api_key, lat, lng) bydaily = forecast.daily() day=bydaily.data[1] temp = temp.append(pd.Series([i.cities,day.time,day.temperatureMin,day.temperatureMax,day.precipProbability,day.humidity,day.windSpeed,day.summary],index=['Cities','Day','Min_Temperature','Max_Temperature','Precipitation_prob','Humidity','Wind_Speed','Summary']),ignore_index=True) # Appending the data for each day to the previous data Weather=Weather.append(temp,ignore_index=True) # Storing the temporary updated dataframe locally Weather.to_csv('Temp_Weather.csv') # Storing the updated weather data in the local file system to the HDFS i.e the Blob Storage from azure.storage.blob import ContentSettings block_blob_service.create_blob_from_path( 'arnabcluster', 'Project/Weather/Weather.csv', 'Temp_Weather.csv', content_settings=ContentSettings(content_type='text/csv') )
#O,O,X,X,X,O,O,O, #O,O,X,O,0,X,O,O, #O,O,X,O,O,0,X,O, #O,O,X,0,0,0,X,O, #O,O,X,O,O,0,X,O, #O,O,X,O,O,0,X,O, #O,O,X,O,0,X,O,O, #O,O,X,X,X,O,O,O] if stink == True: X = (255,0,0) indx = [2,3,4,10,13,18,22,26,30,34,38,42,46,50,53,58,59,60] for j in indx: pixels[j] = X hat.set_pixels(pixels) with open('data.csv', 'a') as data_file: csv_file = csv.writer(data_file, delimiter=',') csv_file.writerow([timestamp, temp, temp_p, hum, pressure]) time.sleep(60) except KeyboardInterrupt: break # Clear sensor and quit hat.clear() block_blob.create_blob_from_path('example', 'final_output.csv', 'data.csv') print '\rSensor Cleared' sys.exit()
class AzureTransfer(BaseTransfer): def __init__(self, account_name, account_key, container_name, prefix=None): # NOTE: Azure wants all paths to start with a slash prefix = "/{}".format(prefix.lstrip("/") if prefix else "") super().__init__(prefix=prefix) self.account_name = account_name self.account_key = account_key self.container_name = container_name self.conn = BlockBlobService(account_name=self.account_name, account_key=self.account_key) self.container = self.get_or_create_container(self.container_name) self.log.debug("AzureTransfer initialized") # XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking #self.log.warning("AzureTransfer is experimental and has not been thoroughly tested") def get_metadata_for_key(self, key): key = self.format_key_for_backend(key) return self._list_blobs(key)[0]["metadata"] def _metadata_for_key(self, key): return self._list_blobs(key)[0]["metadata"] def list_path(self, key): path = self.format_key_for_backend(key, trailing_slash=True) return self._list_blobs(path) def _list_blobs(self, path): self.log.debug("Listing path %r", path) items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata") result = [] for item in items: result.append({ "last_modified": item.properties.last_modified.isoformat(), "metadata": item.metadata, "name": self.format_key_from_backend(item.name), "size": item.properties.content_length, }) return result def delete_key(self, key): key = self.format_key_for_backend(key) self.log.debug("Deleting key: %r", key) return self.conn.delete_blob(self.container_name, key) def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=None): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to) meta = self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to) if progress_callback: progress_callback(1, 1) return meta def get_contents_to_fileobj(self, key, fileobj_to_store_to, *, progress_callback=None): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) meta = self.conn.get_blob_to_file(self.container_name, key, fileobj_to_store_to) if progress_callback: progress_callback(1, 1) return meta def get_contents_to_string(self, key): key = self.format_key_for_backend(key) self.log.debug("Starting to fetch the contents of: %r", key) return self.conn.get_blob_to_bytes(self.container_name, key), self._metadata_for_key(key) def store_file_from_memory(self, key, memstring, metadata=None): key = self.format_key_for_backend(key) self.conn.create_blob_from_bytes(self.container_name, key, memstring, x_ms_meta_name_values=self.sanitize_metadata(metadata)) def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): key = self.format_key_for_backend(key) self.conn.create_blob_from_path(self.container_name, key, filepath,None) # x_ms_meta_name_values=self.sanitize_metadata(metadata)) def get_or_create_container(self, container_name): start_time = time.time() self.conn.create_container(container_name) self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time) return container_name
def file_uploaded(self, botengine, device_object, file_id, filesize_bytes, content_type, file_extension): """ A device file has been uploaded :param botengine: BotEngine environment :param device_object: Device object that uploaded the file :param file_id: File ID to reference this file at the server :param filesize_bytes: The file size in bytes :param content_type: The content type, for example 'video/mp4' :param file_extension: The file extension, for example 'mp4' """ # We are demonstrating video processing here, so avoid video processing on files that are not videos. if "video" not in content_type: botengine.get_logger().info("The uploaded file is not a video, skipping processing ...") return # Create full file path FILE_NAME = "video." + file_extension FILE_PATH= os.path.dirname(os.path.abspath(__file__)) FULL_FILE_PATH =os.path.join(FILE_PATH, FILE_NAME) # Download file to local device botengine.download_file(file_id, FILE_NAME) # Create Blob service and upload file to Microsoft Blob Storage block_blob_service = BlockBlobService(account_name='andresdemo', account_key='6kG0iguGEK9h41shJasZOW4v/uPKr1Guu8RFWfLhWf7MnDcJaaw5eFhhYGSqaFk2qpqX8JIpJobe0bY8MmYa+g==') block_blob_service.create_container(CONTAINER_NAME) # Set Public Access to container # A try and except block is used due to a occassional logger exception (Doesn't imapact function) try: block_blob_service.set_container_acl(CONTAINER_NAME, public_access=PublicAccess.Container) except: pass # Convert the file into a blob and store it in Microsoft Azure Blob Storage # A try and except block is used due to a occassional logger exception (Doesn't imapact function) try: block_blob_service.create_blob_from_path(CONTAINER_NAME, FILE_NAME, FULL_FILE_PATH) except: pass # Get Video URL url = "https://" + ACCOUNT_NAME + ".blob.core.windows.net/" + CONTAINER_NAME + "/" + FILE_NAME # Get Access Token token = ""; headers = { 'Ocp-Apim-Subscription-Key': API_KEY, } params = urllib.urlencode({ 'allowEdit': 'True', }) # HTTP GET request to Video Indexer API to acquire access token try: conn = httplib.HTTPSConnection('api.videoindexer.ai') conn.request("GET", "/auth/" + ACCOUNT_LOCATION + "/Accounts/" + ACCOUNT_ID + "/AccessToken?%s" % params, headers=headers) response = conn.getresponse() token = response.read() token = token[1:len(token)-1] conn.close() except Exception as e: print("[Errno {0}] {1}".format(e.errno, e.strerror)) # Use Access Token to upload Video file headers = { 'Content-Type': 'multipart/form-data', } params = urllib.urlencode({ 'Content-Type': 'multipart/form-data', 'videoUrl': url, 'streamingPreset': 'Default', 'privacy': "Public" }) try: conn = httplib.HTTPSConnection('api.videoindexer.ai') conn.request("POST", "/" + ACCOUNT_LOCATION + "/Accounts/" + ACCOUNT_ID + "/Videos?accessToken=" +token + "&name=Sample&%s" % params, headers=headers) response = conn.getresponse() data = response.read() d = json.loads(data) conn.close() except Exception as e: print("[Errno {0}] {1}".format(e.errno, e.strerror)) botengine.get_logger().info('Video Processing..') # Get Video Index params = urllib.urlencode({ 'accessToken': token, 'language': 'English', }) conn = httplib.HTTPSConnection('api.videoindexer.ai') conn.request("GET", "/" + ACCOUNT_LOCATION + "/Accounts/" + ACCOUNT_ID + "/Videos/"+d["id"]+"/Index?%s" % params) response = conn.getresponse() data = response.read() result = json.loads(data) # Set a timer to queue for results # Passing in Video ID and Access Token self.start_timer_s(botengine, 5, [d["id"], token]) return
from azure.storage.blob import BlockBlobService from azure.storage.blob import PublicAccess from azure.storage.blob import ContentSettings AzureStorageAccount = "istagingstorage" AzureStorageAccessKey = "qkFU/ah2v4cHvQ7oAZASb2HRGFUkJhg2xs5KBYB+2fEnYmSp6hZH9U3vEO6TujzHHdBF3HWVgqalwcUuvIBMUQ==" AzureStorageContainerName = "peter-container" block_blob_service = BlockBlobService(account_name=AzureStorageAccount, account_key=AzureStorageAccessKey) block_blob_service.create_blob_from_path( AzureStorageContainerName, 'out.jpg', 'img.jpg', content_settings=ContentSettings(content_type='image/png') )
if __name__ == '__main__': parms = sys.argv input_block_blob_service = BlockBlobService(account_name=parms[1], sas_token=parms[2], protocol='https') input_block_blob_service.get_blob_to_path(parms[3], parms[4], 'input.txt') input_file=open("input.txt","r") wordcount={} for word in input_file.read().split(): if word not in wordcount: wordcount[word] = 1 else: wordcount[word] += 1 input_file.close() now = datetime.datetime.now(pytz.timezone("Asia/Tokyo")) fmt = "%Y-%m-%d-%H-%M-%S-%Z" output_filename = now.strftime(fmt) + ".txt" output_file=open(output_filename,"w") for k,v in wordcount.items(): output_file.writelines(k + " " + str(v) + "\n") output_file.close() output_block_blob_service = BlockBlobService(account_name=parms[5], sas_token=parms[7], protocol='https') output_block_blob_service.create_blob_from_path(parms[6], output_filename, output_filename)
#!/usr/bin/python import sys,os from azure.storage.blob import BlockBlobService from azure.storage.blob import ContentSettings block_blob_service = BlockBlobService(account_name=str(sys.argv[1]), account_key=str(sys.argv[2])) block_blob_service.create_container('keys') block_blob_service.create_blob_from_path( 'keys', str(sys.argv[3]), os.path.join(os.getcwd(),str(sys.argv[3])), content_settings=ContentSettings(content_type='text') )