def test_add_block_to_append_blob_with_container_sas(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange token = self.bs.generate_container_shared_access_signature( self.container_name, permission=ContainerPermissions.READ + ContainerPermissions.CREATE + ContainerPermissions.ADD, expiry=datetime.utcnow() + timedelta(hours=1), ) bs_with_sas = AppendBlobService( account_name=self.settings.STORAGE_ACCOUNT_NAME, sas_token=token, protocol=self.settings.PROTOCOL) # Act test_blob_name = "testblobname" text_blob_content = "test-blob-content" bs_with_sas.create_blob(self.container_name, blob_name=test_blob_name) bs_with_sas.append_blob_from_text(self.container_name, blob_name=test_blob_name, text=text_blob_content) blob_content = bs_with_sas.get_blob_to_text(self.container_name, blob_name=test_blob_name) # Assert self.assertEqual(blob_content.content, text_blob_content)
def update_status(self, status=0, errorMsg=None): if self.files is not None: text = str(status) + '&,&' + datetime.datetime.utcnow().strftime( '%a, %d %b %Y %H:%M:%S GMT') + '&,&' + '&,&'.join(self.files) else: text = str(status) + '&,&' + datetime.datetime.utcnow().strftime( '%a, %d %b %Y %H:%M:%S GMT') if errorMsg is not None: text = str(status) + '&,&' + datetime.datetime.utcnow().strftime( '%a, %d %b %Y %H:%M:%S GMT') + '&,&' + errorMsg name = self.folderName + '/status.cvs' append_blob_service = AppendBlobService( account_name='navview', account_key= '+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==', protocol='http') append_blob_service.create_blob( container_name='data', blob_name=name, content_settings=ContentSettings(content_type='text/plain')) append_blob_service.append_blob_from_text(container_name='data', blob_name=name, text=text) #if __name__ == '__main__': #DataUpload(dirPath='../../demo_saved_data/2018-07-25-13-01-51').begin_update_files()
def WriteLog( txt ): """ ログファイルにテキストを出力する。末尾に改行コードが追加される。 """ szRet = ""; if( DEBUG_MODE ): print( r"{0:%Y-%m-%d %H:%M:%S}".format( datetime.datetime.now() ) + r" : " + txt + "\r\n" ); return( "Debug モードのためスキップしました。" ); try: szRet = "AppendBlobService"; blob_service = AppendBlobService( account_name, account_key ); szRet = "append_blob_from_text"; blob_service.append_blob_from_text( log_container_name, log_file_name, r"{0:%Y-%m-%d %H:%M:%S}".format( datetime.datetime.now() ) + r" : " + txt + "\r\n" ) szRet = "OK"; except: #szRet = "Log exception"; pass; #try return szRet;
def append_log(today): """ ログ情報をblobにあげる。事前にblobアカウント・blob containerを作っておくこと :param today: :return: """ print('append log') append_blob_service = AppendBlobService(account_name=account_name, account_key=account_key) append_blob_service.append_blob_from_text(container_name, blob_name, 'test' + '\n')
def azure_blob_storage(): try: # To create a blob service or connection to azure blob storage block_blob_service = BlockBlobService(account_name='account_name', account_key='account_key') # To create a container container_name ='container_name' block_blob_service.create_container(container_name) # Set access to a container such as public ,readonly,private block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container) # Create a file to upload to azure blob storage local_file_name ="Test.txt" path_to_file =os.path.join('local_path', local_file_name) # To write to the file local_file = open(full_path_to_file, 'w') local_file.write("hi peoplee") local_file.close() # Upload the created file, use local_file_name for the blob name block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file) # To List all the blobs in the container container_List = block_blob_service.list_blobs(container_name) for file in container_List: print("Blob name: "+{file.name}) # Download the blob(s). download_file_path = os.path.join('local_path', 'local_file_name') block_blob_service.get_blob_to_path(container_name, local_file_name, download_file_path) #delete a blob block_blob_service.delete_blob('container_name', 'blob_name') #Append to a blob service append_blob_service = AppendBlobService(account_name='myaccount', account_key='mykey') # The same containers can hold all types of blobs append_blob_service.create_container('container_name') #To append file must exists append_blob_service.create_blob('container_name', 'append_blob') #Append to a blob service append_blob_service.append_blob_from_text('container_name', 'append_blob', 'Hello, world!') append_blob = append_blob_service.get_blob_to_text('container_name', 'append_blob') # Clean up resources. This includes the container and the temp files block_blob_service.delete_container(container_name) except Exception as e:
def WriteLog(txt): """ ログファイルにテキストを出力する。末尾に改行コードが追加される。 """ szRet = "" try: szRet = "AppendBlobService" blob_service = AppendBlobService(account_name, account_key) szRet = "append_blob_from_text" blob_service.append_blob_from_text( log_container_name, log_file_name, r"{0:%Y-%m-%d %H:%M:%S}".format(datetime.datetime.now()) + r" : " + txt + "\r\n") szRet = "OK" except: #szRet = "Log exception"; pass return szRet
class AppendBlob: '''Append blob used to append data, each time called append, the content will be append to the blob''' def __init__(self): #account = account_name or secretconf["azure"]["storage"][0]["account"] #key = account_key or secretconf["azure"]["storage"][0]["key"] connstr = os.getenv( "AZURE_STORAGE_CONNECTION_STRING", False) or secretconf["azure"]["storage"][0]["connection_string"] self.abservice = AppendBlobService(connection_string=connstr) def create(self, container, blob, metadata=None): '''Create an empty blob Args: container: name of the container blob: name of the blob, use '/' to create a folder metadata: meta data (dict object, value must be str) of the text Returns: url of blob ''' self.abservice.create_blob(container, blob, metadata=metadata, content_settings=textcontent, if_none_match="*") now = datetime.now() start = now + timedelta(-1, 0, 0) expiry = now + timedelta(365, 0, 0) sastoken = self.abservice.generate_blob_shared_access_signature( container, blob, permission=blobpermission, expiry=expiry, start=start) return self.abservice.make_blob_url(container, blob, sas_token=sastoken) def appendText(self, container, blob, text, metadata=None): '''Append text to blob''' self.abservice.append_blob_from_text(container, blob, text)
class LogIMU380Data: def __init__(self, imu, user): '''Initialize and create a CSV file ''' self.name = 'data-' + datetime.datetime.now().strftime( '%Y_%m_%d_%H_%M_%S') + '.csv' self.file = open('data/' + self.name, 'w') self.first_row = 0 # decode converts out of byte array self.sn = imu.device_id.split(" ")[0] self.pn = imu.device_id.split(" ")[1] self.device_id = imu.device_id self.odr_setting = imu.odr_setting self.packet_type = imu.packet_type odr_rates = {0: 'Quiet', 1: '100Hz', 2: '50Hz', 4: '25Hz'} self.sample_rate = odr_rates[self.odr_setting] def log(self, data, odr_setting): '''Write row of CSV file based on data received. Uses dictionary keys for column titles ''' if not self.first_row: self.first_row = 1 labels = ''.join('{0:s},'.format(key) for key in data) labels = labels[:-1] header = labels + '\n' else: self.first_row += 1 header = '' str = '' for key in data: if key == 'BITstatus' or key == 'GPSITOW' or key == 'counter' or key == 'timeITOW': str += '{0:d},'.format(data[key]) else: str += '{0:3.5f},'.format(data[key]) str = str[:-1] str = str + '\n' self.file.write(header + str) def write_to_azure(self): # check for internet # if not self.internet_on(): # return False # record file to cloud self.append_blob_service = AppendBlobService( account_name='navview', account_key= '+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==', protocol='http') self.append_blob_service.create_blob( container_name='data', blob_name=self.name, content_settings=ContentSettings(content_type='text/plain')) f = open("data/" + self.name, "r") self.append_blob_service.append_blob_from_text('data', self.name, f.read()) # TODO: check if success # record record to ansplatform self.record_to_ansplatform() def record_to_ansplatform(self): data = { "pn": self.pn, "sn": self.sn, "fileName": self.user['fileName'], "url": self.name, "imuProperties": json.dumps(self.imu_properties), "sampleRate": self.sample_rate, "packetType": self.packet_type, "userId": self.user['id'] } url = "https://ans-platform.azurewebsites.net/api/datafiles/replaceOrCreate" data_json = json.dumps(data) headers = { 'Content-type': 'application/json', 'Authorization': self.user['access_token'] } response = requests.post(url, data=data_json, headers=headers) response = response.json() print(response) # clean up self.file.close() self.name = '' return #ends thread def internet_on(self): try: urllib2.urlopen('https://ans-platform.azurewebsites.net', timeout=1) return True except urllib2.URLError as err: return False def close(self): time.sleep(0.1) threading.Thread(target=self.write_to_azure).start()
FROM Kwinana_Plc_notification t1 join Kwinana_Plc_refference_data t2 on t1.Commodity = t2.Commodity order by AutoId DESC""" def wa_timenow(): #get current perth time dtn = datetime.now() + timedelta(hours=8) return dtn cursor.execute(sql) row = cursor.fetchone() outval = '' if row[0] == None: outval = ",".join([str(i) for i in row[1:]]) #connect to request-log blob and log request append_blob_service = AppendBlobService(account_name=myaccount, account_key=mykey) append_blob_service.append_blob_from_text( 'requestlogs', 'request.txt', "%s,%s " % (wa_timenow(), outval)) append_blob_service.append_blob_from_text('requestlogs', 'request.txt', "\n") append_blob = append_blob_service.get_blob_to_text('requestlogs', 'request.txt') #send response response = open(os.environ['res'], 'w') response.write(outval) response.close()
# generator = block_blob_service.list_blobs('mycontainer') # for blob in generator: # print(blob.name) # block_blob_service.get_blob_to_path('mycontainer', 'myblockblob', 'out-sunset.png') # block_blob_service.delete_blob('mycontainer', 'myblockblob') from azure.storage.blob import AppendBlobService append_blob_service = AppendBlobService(account_name='bobur', account_key='6e60FZapOXAmUbFBw0SpE1lHRP3RkXOMYRaalWmRBoz4+xI5tvjaJzxXuYyt+yfWxjPXpz5X3PmyIFiQmSkjbw==') # The same containers can hold all types of blobs append_blob_service.create_container('myappendcontainer') # Append blobs must be created before they are appended to append_blob_service.create_blob('myappendcontainer', 'myappendblob') append_blob_service.append_blob_from_text('myappendcontainer', 'myappendblob', u'Hello, world!') append_blob = append_blob_service.get_blob_to_text('myappendcontainer', 'myappendblob') print append_blob.content
class FileLoger(): def __init__(self): '''Initialize and create a CSV file ''' start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') self.rover_properties = utility.load_configuration(os.path.join('setting', 'rover.json')) if not self.rover_properties: os._exit(1) if not os.path.exists('data/'): os.mkdir('data/') self.output_packets = self.rover_properties['userMessages']['outputPackets'] self.log_file_rows = {} self.log_file_names = {} self.log_files_obj = {} self.log_files = {} self.user_file_name = '' # the prefix of log file name. self.msgs_need_to_log = [] self.ws = False # azure app. self.user_id = '' self.file_name = '' self.sas_token = '' self.db_user_access_token = '' self.host_url = self.rover_properties['userConfiguration']['hostURL'] # self.threads = [] # thread of receiver and paser self.exit_thread = False # flag of exit threads self.exit_lock = threading.Lock() # lock of exit_thread self.data_dict = {} # data container self.data_lock = threading.Lock() # lock of data_queue def start_user_log(self, file_name='', ws=False): ''' start log. return: 0: OK 1: exception that has started logging already. 2: other exception. ''' try: if len(self.log_file_rows) > 0: return 1 # has started logging already. self.ws = ws self.exit_thread = False self.user_file_name = file_name start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') for packet in self.output_packets: if 1 == packet['save2file']: self.msgs_need_to_log.append(packet['name']) self.log_file_rows[packet['name']] = 0 if self.user_file_name == '': self.log_file_names[packet['name']] = packet['name'] +'-' + start_time + '.csv' else: self.log_file_names[packet['name']] = self.user_file_name + '_' + packet['name'] +'-' + start_time + '.csv' self.log_files[packet['name']] = self.log_file_names[packet['name']] self.log_files_obj[packet['name']] = open('data/' + self.log_file_names[packet['name']], 'w') if self.ws: self.get_sas_token() self.data_dict.clear() for i, (k, v) in enumerate(self.log_files.items()): # k:pack type v:log file name self.data_dict[v]='' threading.Thread(target=self.upload_azure, args=(k,v)).start() return 0 except Exception as e: print('Exception! File:[{0}], Line:[{1}]. Exception:{2}'.format(__file__, sys._getframe().f_lineno, e)) return 2 def stop_user_log(self): ''' stop log. return: 0: OK 1: exception that driver hasn't started logging files yet. 2: other exception. ''' rev = 0 try: if len(self.log_file_rows) == 0: return 1 # driver hasn't started logging files yet. for i, (k, v) in enumerate(self.log_files_obj.items()): v.close() self.log_file_rows.clear() self.log_file_names.clear() self.log_files_obj.clear() rev = 0 except Exception as e: print(e) rev = 2 if self.ws: time.sleep(1) self.exit_lock.acquire() self.exit_thread = True self.exit_lock.release() self.ws = False return rev # def upload_callback(self, current, total): # print('({}, {})'.format(current, total)) def upload_azure(self, packet_type, log_file_name): if self.db_user_access_token == '' or self.sas_token == '': print("Error: Can not upload log to azure since token is empty! Please check the network.") print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:'), log_file_name, ' start.') accountName = 'navview' countainerName = 'data-1000' fileName = log_file_name bcreate_blob_ok = False error_connection = 'ConnectionError' error_authorization = 'AuthenticationFailed' ii=0 while True: # get data from data_dict. self.data_lock.acquire() text = self.data_dict[log_file_name] self.data_dict[log_file_name] = '' self.data_lock.release() # check if user stop logging data. self.exit_lock.acquire() if self.exit_thread: # check for internet and text if text == '' or (not self.internet_on()): self.exit_lock.release() break else: pass self.exit_lock.release() #let CPU have a break. if text == '' : time.sleep(1) continue #create blob on azure if not bcreate_blob_ok: try: self.append_blob_service = AppendBlobService(account_name=accountName, sas_token=self.sas_token, protocol='http') self.append_blob_service.create_blob(container_name=countainerName, blob_name=fileName, content_settings=ContentSettings(content_type='text/plain')) bcreate_blob_ok = True threading.Thread(target=self.save_to_db_task, args=(packet_type, log_file_name)).start() except Exception as e: # print('Exception when create_blob:', type(e), e) if error_connection in str(e): pass elif error_authorization in str(e): self.get_sas_token() self.append_blob_service = AppendBlobService(account_name=accountName, sas_token=self.sas_token, protocol='http') print('Retry to create_blob again...') continue # append blob on azure try: # self.append_blob_service.append_blob_from_text(countainerName, fileName, text, progress_callback=self.upload_callback) self.append_blob_service.append_blob_from_text(countainerName, fileName, text) except Exception as e: # print('Exception when append_blob:', type(e), e) if error_connection in str(e): pass elif error_authorization in str(e): self.get_sas_token() self.append_blob_service = AppendBlobService(account_name=accountName, sas_token=self.sas_token, protocol='http') # if append blob failed, do not drop 'text', but push 'text' to data_dict and re-append next time. self.data_lock.acquire() self.data_dict[log_file_name] = text + self.data_dict[log_file_name] self.data_lock.release() if bcreate_blob_ok: # if not self.save_to_ans_platform(packet_type, log_file_name): # print('save_to_ans_platform failed.') print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:') , log_file_name, ' done.') def save_to_db_task(self, packet_type, file_name): if not self.save_to_ans_platform(packet_type, file_name): print('save_to_ans_platform failed.') def update(self, packet, packet_type, is_var_len_frame): if len(self.log_file_rows) == 0: #if hasn't started logging. return if packet_type in self.msgs_need_to_log: if is_var_len_frame: self.log_var_len(packet, packet_type) else: self.log(packet, packet_type) def get_log_file_names(self): return self.log_file_names.copy() def log(self, data, packet_type): ''' Parse the data, read in from the unit, and generate a data file using the json properties file to create a header and specify the precision of the data in the resulting data file. ''' output_packet = next((x for x in self.output_packets if x['name'] == packet_type), None) '''Write row of CSV file based on data received. Uses dictionary keys for column titles ''' if self.log_file_rows[packet_type] == 0: # Loop through each item in the data dictionary and create a header from the json # properties that correspond to the items in the dictionary labels = '' # for key in data: for i, (k, v) in enumerate(data.items()): '''dataStr = output_packet['payload'][i]['name'] + \ ' [' + \ output_packet['payload'][i]['unit'] + \ ']''' dataStr = output_packet['payload'][i]['name'] unitStr = output_packet['payload'][i]['unit'] if unitStr == '': labels = labels + '{0:s},'.format(dataStr) else: labels = labels + '{0:s} ({1:s}),'.format(dataStr, unitStr) # Remove the comma at the end of the string and append a new-line character labels = labels[:-1] header = labels + '\n' else: header = '' self.log_file_rows[packet_type] += 1 # Loop through the items in the data dictionary and append to an output string # (with precision based on the data type defined in the json properties file) str = '' for i, (k, v) in enumerate(data.items()): outputPcktType = output_packet['payload'][i]['type'] if outputPcktType == 'uint32' or outputPcktType == 'int32' or \ outputPcktType == 'uint16' or outputPcktType == 'int16' or \ outputPcktType == 'uint64' or outputPcktType == 'int64': # integers and unsigned integers str += '{0:d},'.format(v) elif outputPcktType == 'double': # double str += '{0:0.8f},'.format(v)# 15.12 elif outputPcktType == 'float': str += '{0:0.4f},'.format(v) # 12.8 elif outputPcktType == 'uint8': # byte str += '{0:d},'.format(v) elif outputPcktType == 'uchar' or outputPcktType == 'char' or outputPcktType == 'string': # character str += '{:},'.format(v) else: # unknown str += '{0:3.5f},'.format(v) # str = header + str[:-1] + '\n' self.log_files_obj[packet_type].write(str) self.log_files_obj[packet_type].flush() if self.ws: self.data_lock.acquire() self.data_dict[self.log_files[packet_type]] = self.data_dict[self.log_files[packet_type]] + str self.data_lock.release() def log_var_len(self, data, packet_type): ''' Parse the data, read in from the unit, and generate a data file using the json properties file to create a header and specify the precision of the data in the resulting data file. ''' output_packet = next((x for x in self.output_packets if x['name'] == packet_type), None) '''Write row of CSV file based on data received. Uses dictionary keys for column titles ''' if self.log_file_rows[packet_type] == 0: # Loop through each item in the data dictionary and create a header from the json # properties that correspond to the items in the dictionary labels = '' for value in output_packet['payload']: dataStr = value['name'] unitStr = value['unit'] if unitStr == '': labels = labels + '{0:s},'.format(dataStr) else: labels = labels + '{0:s} ({1:s}),'.format(dataStr, unitStr) # Remove the comma at the end of the string and append a new-line character labels = labels[:-1] header = labels + '\n' else: header = '' self.log_file_rows[packet_type] += 1 # Loop through the items in the data dictionary and append to an output string # (with precision based on the data type defined in the json properties file) str = '' const_str = '' var_str = '' var_fileld_tpyes = [] var_fileld_num = len(output_packet['payload']) - output_packet['var_num']['field_idx'] const_fileld_num = len(output_packet['payload']) - var_fileld_num for idx, value in enumerate(output_packet['payload']): if idx >= const_fileld_num: var_fileld_tpyes.append(value['type']) for idx, key in enumerate(data): if idx == 0: # handle const filelds which are all in the first item of data. for i, (k, v) in enumerate(key.items()): outputPcktType = output_packet['payload'][i]['type'] if outputPcktType == 'uint32' or outputPcktType == 'int32' or \ outputPcktType == 'uint16' or outputPcktType == 'int16' or \ outputPcktType == 'uint64' or outputPcktType == 'int64': # integers and unsigned integers const_str += '{0:d},'.format(v) elif outputPcktType == 'double': # double const_str += '{0:0.12f},'.format(v) # 15.12 elif outputPcktType == 'float': const_str += '{0:0.4f},'.format(v) # 12.8 elif outputPcktType == 'uint8': # byte const_str += '{0:d},'.format(v) elif outputPcktType == 'uchar' or outputPcktType == 'char' or outputPcktType == 'string': # character const_str += '{:},'.format(v) else: # unknown const_str += '{0:3.5f},'.format(v) else: for i, (k, v) in enumerate(key.items()): outputPcktType = var_fileld_tpyes[i] if outputPcktType == 'uint32' or outputPcktType == 'int32' or \ outputPcktType == 'uint16' or outputPcktType == 'int16' or \ outputPcktType == 'uint64' or outputPcktType == 'int64': # integers and unsigned integers var_str += '{0:d},'.format(v) elif outputPcktType == 'double': # double var_str += '{0:15.12f},'.format(v)# 15.12 elif outputPcktType == 'float': var_str += '{0:12.4f},'.format(v) # 12.8 elif outputPcktType == 'uint8': # byte var_str += '{0:d},'.format(v) elif outputPcktType == 'uchar' or outputPcktType == 'char': # character var_str += '{:},'.format(v) else: # unknown var_str += '{0:3.5f},'.format(v) str = const_str + var_str str = header + str[:-1] + '\n' self.log_files_obj[packet_type].write(str) self.log_files_obj[packet_type].flush() if self.ws: self.data_lock.acquire() self.data_dict[self.log_files[packet_type]] = self.data_dict[self.log_files[packet_type]] + str self.data_lock.release() header = '' str = '' var_str = '' def set_user_id(self, user_id): self.user_id = user_id if not isinstance(self.user_id, str): self.user_id = str(self.user_id) def set_user_access_token(self, access_token): self.db_user_access_token = access_token def upload(self, log_file_names): t = threading.Thread(target=self.upload_to_azure_task, args=(log_file_names, )) t.start() def upload_to_azure_task(self, log_files_dict): self.get_sas_token() if self.db_user_access_token != '' and self.sas_token != '': print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:') , 'Start.') for i, (k, v) in enumerate(log_files_dict.items()): # k: packet type; v: log file name print('upload:', v) self.upload_to_azure(k, v) # self.db_user_access_token = '' # self.sas_token = '' print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:') , 'Done.') def get_sas_token(self): try: url = self.host_url + "token/storagesas" headers = {'Content-type': 'application/json', 'Authorization': self.db_user_access_token} response = requests.post(url, headers=headers) rev = response.json() if 'token' in rev: self.sas_token = rev['token'] else: self.sas_token = '' print('Error: Get sas token failed!') except Exception as e: print('Exception when get_sas_token:', e) def upload_to_azure(self, packet_type, file_name): ''' Upload CSV's to Azure container. ''' f = open("data/" + file_name, "r") text = f.read() #.decode("utf-8") try: self.azure_storage('navview', self.sas_token, 'data', file_name, text) except Exception as e: print('azure_storage exception:', e) return # Try again! # self.azure_storage('navview', self.sas_token, 'data', file_name, text) pass ''' Trigger Database upload ''' rev = self.save_to_ans_platform(packet_type, file_name) if not rev: print('save_to_ans_platform failed.') def azure_storage(self, accountName, sasToken, countainerName,fileName,text): if 0: self.append_blob_service = AppendBlobService(account_name=accountName, sas_token=sasToken, protocol='http') self.append_blob_service.create_blob(container_name=countainerName, blob_name=fileName, content_settings=ContentSettings(content_type='text/plain')) self.append_blob_service.append_blob_from_text(countainerName, fileName, text) else: self.block_blob_service = BlockBlobService(account_name=accountName, sas_token=sasToken, protocol='http') self.block_blob_service.create_blob_from_text( container_name= countainerName, blob_name= fileName, text=text, content_settings=ContentSettings(content_type='text/plain')) def save_to_ans_platform(self, packet_type, file_name): ''' Upload CSV related information to the database. ''' try: data = {"type": 'INS', "model": 'INS1000', "fileName": file_name, "url": file_name, "userId": self.user_id, "logInfo": { "pn": '11', "sn": '', "packetType":packet_type,"insProperties":json.dumps(self.rover_properties)}} url = self.host_url + "api/recordLogs/post" data_json = json.dumps(data) headers = {'Content-type': 'application/json', 'Authorization': self.db_user_access_token} response = requests.post(url, data=data_json, headers=headers) return True if 'success' in response.json() else False except Exception as e: print('Exception when update db:', e) def internet_on(self): try: url = 'https://navview.blob.core.windows.net/' if sys.version_info[0] > 2: import urllib.request response = urllib.request.urlopen(url, timeout=1) else: import urllib2 response = urllib2.urlopen(url, timeout=1) # print(response.read()) return True except urllib2.URLError as err: return False
def main(msg: func.QueueMessage) -> None: # consume videoname from event that is put on blob storage using filename logging.info('Python queue trigger function processed a queue item: %s', msg.get_body().decode('utf-8')) raw = msg.get_body().decode('utf-8') logging.info(raw) eventVideo = json.loads(raw) blockBlobService = BlockBlobService( account_name=os.environ['remoteStorageAccountName'], account_key=os.environ['remoteStorageAccountKey']) appendBlobService = AppendBlobService( account_name=os.environ['remoteStorageAccountName'], account_key=os.environ['remoteStorageAccountKey']) cap = getVideo(blockBlobService, eventVideo) folder = eventVideo["filename"] # process video x = 1 frameRate = cap.get(5) #frame rate numberOfPicturesPerSecond = int(os.environ["numberOfPicturesPerSecond"]) # start creating frames from video while (cap.isOpened()): frameId = cap.get(1) #current frame number ret, frame = cap.read() if (ret != True): break # in case frame matches a multiple of the frame, create image if frameId % math.floor(frameRate / numberOfPicturesPerSecond) == 0: logging.info("create cap" + str(x)) # convert frame to PIL image frame_conv = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) pilImage = Image.fromarray(frame_conv) imgByteArr = BytesIO() pilImage.save(imgByteArr, format='PNG') imgByteArr = imgByteArr.getvalue() # write image to blob for logging imageFileName = folder + "/log/image" + str(int(x)) + "_img.png" blockBlobService.create_blob_from_bytes( os.environ['remoteStorageOutputContainer'], imageFileName, imgByteArr) # try to recognize objects and text from image description, caption, confidence, text, fullText = analyzeImage( imgByteArr) # in case trainnumber is recognized of train, public to Power BI trainNumber = "" if (len(text) == 4 and text.isdigit()) or text[:2] == "NL": trainNumber = text publishPowerBI(blockBlobService, x, imgByteArr, folder, eventVideo, text, caption) # write identification of image to csv loggingCsv = "\n" + str(imageFileName) + "|" + str( caption) + "|" + str(confidence) + "|" + str( json.dumps(text).replace( '|', ':pipe')) + "|" + str(trainNumber) + "|" + str( json.dumps(fullText).replace('|', ':pipe')) appendBlobService.append_blob_from_text("logging", os.environ["loggingcsv"], loggingCsv) # increment image x += 1
# Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer append_blob_service = AppendBlobService(account_name='storage_account_name', account_key='storage_account_key') # Reads the start and stop index passed in through SLURM start = int(sys.argv[1]) stop = int(sys.argv[2]) #Creates the blob for this batch. append_blob_service.create_blob('distances', str(start) + "-" + str(stop) + '.csv') #Logs the start time append_blob_service.append_blob_from_text( 'distances', 'log.txt', "Starting " + str(start) + "-" + str(stop) + ":" + datetime.datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n") LatLongDict = {} # radius of earth in miles R = 3959.0 # Reads the airport data in to a list for easy access. with open('airports-world.csv') as csvfile: reader = csv.DictReader(csvfile) for row in reader: LatLongDict[row['LocationID']] = [row['Latitude'], row['Longitude']] #Creates the column names for the distance table fieldnames = "id" for code1 in LatLongDict:
class RoverLogApp(rover_application_base.RoverApplicationBase): def __init__(self, user=False): '''Initialize and create a CSV file ''' if user and list(user.keys())[0] == 'startLog': self.username = user['startLog']['username'] self.userId = user['startLog']['id'] self.userFilename = user['startLog']['fileName'] self.userAccessToken = user['startLog']['access_token'] self.start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') if not self.load_configuration(): os._exit(1) if not os.path.exists('data/'): os.mkdir('data/') self.output_packets = self.rover_properties['userMessages'][ 'outputPackets'] self.first_row = {} self.log_file_names = {} self.log_files = {} if user: with open('tempLogFiles.json', 'r') as outfile: data = json.load(outfile) self.log_file_names = data if not user: with open('tempLogFiles.json', 'w') as outfile: json.dump({}, outfile) try: for packet in self.output_packets: self.first_row[packet['name']] = 0 self.log_file_names[packet['name']] = packet[ 'name'] + '-' + self.start_time + '.csv' self.log_files[packet['name']] = open( 'data/' + self.log_file_names[packet['name']], 'w') # just log Compact Navigation Message entry = { packet['name']: self.log_file_names[packet['name']] } with open('tempLogFiles.json') as f: data = json.load(f) data.update(entry) with open('tempLogFiles.json', 'w') as f: json.dump(data, f) except: pass if user and list(user.keys())[0] == 'startLog': self.savetoAnsPlatform() if user and list(user.keys())[0] == 'stopLog': time.sleep(10) self.close() # os.remove("tempLogFiles.json") def on_reinit(self): print("RoverLogApp.on_reinit()") pass # Is it necessary to create a new file to log when replug serial connector? # self.start_time = datetime.datetime.now().strftime('%Y%m%d_%H_%M_%S') # try: # for packet in self.output_packets: # self.first_row[packet['name']] = 0 # self.log_file_names[packet['name']] = packet['name'] +'-' + self.start_time + '.csv' # self.log_files[packet['name']] = open('data/' + self.log_file_names[packet['name']], 'w')# just log Compact Navigation Message # except: # pass def on_find_active_rover(self): print("RoverLogApp.on_find_active_rover()") def on_message(self, *args): packet_type = args[0] self.data = args[1] is_var_len_frame = args[2] if is_var_len_frame: self.log_var_len(self.data, packet_type) else: self.log(self.data, packet_type) def on_exit(self): pass def load_configuration(self): ''' load properties from 'rover.json' returns: True when load successfully. False when load failed. ''' try: with open('setting/rover.json') as json_data: self.rover_properties = json.load(json_data) return True # except (ValueError, KeyError, TypeError) as error: except Exception as e: print(e) return False def log(self, data, packet_type): ''' Parse the data, read in from the unit, and generate a data file using the json properties file to create a header and specify the precision of the data in the resulting data file. ''' if not self.rover_properties: return output_packet = next( (x for x in self.rover_properties['userMessages']['outputPackets'] if x['name'] == packet_type), None) '''Write row of CSV file based on data received. Uses dictionary keys for column titles ''' if not self.first_row[packet_type]: self.first_row[packet_type] = 1 # Loop through each item in the data dictionary and create a header from the json # properties that correspond to the items in the dictionary labels = '' keyIdx = -1 for key in data: keyIdx = keyIdx + 1 '''dataStr = output_packet['payload'][keyIdx]['name'] + \ ' [' + \ output_packet['payload'][keyIdx]['unit'] + \ ']''' dataStr = output_packet['payload'][keyIdx]['name'] unitStr = output_packet['payload'][keyIdx]['unit'] if unitStr == '': labels = labels + '{0:s},'.format(dataStr) else: labels = labels + '{0:s}({1:s}),'.format(dataStr, unitStr) # Remove the comma at the end of the string and append a new-line character labels = labels[:-1] header = labels + '\n' else: self.first_row[packet_type] += 1 header = '' # Loop through the items in the data dictionary and append to an output string # (with precision based on the data type defined in the json properties file) str = '' keyIdx = -1 for key in data: keyIdx = keyIdx + 1 outputPcktType = output_packet['payload'][keyIdx]['type'] if outputPcktType == 'uint32' or outputPcktType == 'int32' or \ outputPcktType == 'uint16' or outputPcktType == 'int16' or \ outputPcktType == 'uint64' or outputPcktType == 'int64': # integers and unsigned integers str += '{0:d},'.format(data[key]) elif outputPcktType == 'double': # double str += '{0:15.12f},'.format(data[key]) elif outputPcktType == 'float': # print(3) #key + str(2)) str += '{0:12.8f},'.format(data[key]) elif outputPcktType == 'uint8': # byte str += '{0:d},'.format(data[key]) elif outputPcktType == 'uchar' or outputPcktType == 'char': # character str += '{:},'.format(data[key]) else: # unknown str += '{0:3.5f},'.format(data[key]) # str = str[:-1] str = str + '\n' self.log_files[packet_type].write(header + str) def log_var_len(self, data, packet_type): ''' Parse the data, read in from the unit, and generate a data file using the json properties file to create a header and specify the precision of the data in the resulting data file. ''' if not self.rover_properties: return output_packet = next( (x for x in self.rover_properties['userMessages']['outputPackets'] if x['name'] == packet_type), None) '''Write row of CSV file based on data received. Uses dictionary keys for column titles ''' if not self.first_row[packet_type]: self.first_row[packet_type] = 1 # Loop through each item in the data dictionary and create a header from the json # properties that correspond to the items in the dictionary labels = '' for value in output_packet['payload']: dataStr = value['name'] unitStr = value['unit'] if unitStr == '': labels = labels + '{0:s},'.format(dataStr) else: labels = labels + '{0:s}({1:s}),'.format(dataStr, unitStr) # Remove the comma at the end of the string and append a new-line character labels = labels[:-1] header = labels + '\n' else: self.first_row[packet_type] += 1 header = '' # Loop through the items in the data dictionary and append to an output string # (with precision based on the data type defined in the json properties file) str = '' const_str = '' var_str = '' var_fileld_tpyes = [] var_fileld_num = len( output_packet['payload']) - output_packet['var_num']['field_idx'] const_fileld_num = len(output_packet['payload']) - var_fileld_num for idx, value in enumerate(output_packet['payload']): if idx >= const_fileld_num: var_fileld_tpyes.append(value['type']) for idx, key in enumerate(data): if idx < const_fileld_num: outputPcktType = output_packet['payload'][idx]['type'] if outputPcktType == 'uint32' or outputPcktType == 'int32' or \ outputPcktType == 'uint16' or outputPcktType == 'int16' or \ outputPcktType == 'uint64' or outputPcktType == 'int64': # integers and unsigned integers const_str += '{0:d},'.format(list(key.values())[0]) elif outputPcktType == 'double': # double const_str += '{0:15.12f},'.format(list(key.values())[0]) elif outputPcktType == 'float': # print(3) #key + str(2)) const_str += '{0:12.8f},'.format(list(key.values())[0]) elif outputPcktType == 'uint8': # byte const_str += '{0:d},'.format(list(key.values())[0]) elif outputPcktType == 'uchar' or outputPcktType == 'char': # character const_str += '{:},'.format(list(key.values())[0]) else: # unknown const_str += '{0:3.5f},'.format(key.values()[0]) else: idx_key = -1 for k, v in key.items(): idx_key += 1 outputPcktType = var_fileld_tpyes[idx_key] if outputPcktType == 'uint32' or outputPcktType == 'int32' or \ outputPcktType == 'uint16' or outputPcktType == 'int16' or \ outputPcktType == 'uint64' or outputPcktType == 'int64': # integers and unsigned integers var_str += '{0:d},'.format(v) elif outputPcktType == 'double': # double var_str += '{0:15.12f},'.format(v) elif outputPcktType == 'float': # print(3) #key + str(2)) var_str += '{0:12.8f},'.format(v) elif outputPcktType == 'uint8': # byte var_str += '{0:d},'.format(v) elif outputPcktType == 'uchar' or outputPcktType == 'char': # character var_str += '{:},'.format(v) else: # unknown var_str += '{0:3.5f},'.format(v) str = const_str + var_str str = str[:-1] str = str + '\n' self.log_files[packet_type].write(header + str) header = '' str = '' var_str = '' ''' Upload CSV's to Azure container. ''' def uploadtoAzure(self, fileDisplayName): # f = open("data/" + self.user['fileName'], "r") f = open("data/" + fileDisplayName, "r") text = f.read() account_key = '+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==' try: self.azureStorage('navview', account_key, 'data', fileDisplayName, text) except: # Try again! self.azureStorage('navview', account_key, 'data', fileDisplayName, text) ''' Trigger Database upload ''' # self.savetoAnsPlatform() def azureStorage(self, accountName, accountkey, countainerName, fileName, text): self.append_blob_service = AppendBlobService(account_name=accountName, account_key=accountkey, protocol='http') self.append_blob_service.create_blob( container_name=countainerName, blob_name=fileName, content_settings=ContentSettings(content_type='text/plain')) self.append_blob_service.append_blob_from_text(countainerName, fileName, text) ''' Upload CSV related information to the database. ''' def savetoAnsPlatform(self): for files in self.log_file_names: fileDisplayName = files + "-" + self.userFilename + ".csv" data = { "pn": '1.0.0', "sn": 'rtk', "fileName": fileDisplayName, "url": self.log_file_names[files], "imuProperties": json.dumps(self.rover_properties), "sampleRate": '100', "packetType": files, "userId": self.userId } url = "https://api.aceinna.com/api/datafiles/replaceOrCreate" data_json = json.dumps(data) headers = { 'Content-type': 'application/json', 'Authorization': self.userAccessToken } response = requests.post(url, data=data_json, headers=headers) response = response.json() # def close(self,fileName,storedFile): # time.sleep(0.1) # # if self.ws: # storedFile.close() # threading.Thread(target=self.uploadtoAzure(fileName)).start() # # else: # # self.file.close() # # print('close') # # try: # # for packet in self.output_packets: # # self.log_files[packet['name']].close() # # threading.Thread(target=self.write_to_azurelog_files[packet['name']]).start() # # except: # # pass def close(self): for files in self.log_file_names: self.uploadtoAzure(self.log_file_names[files])
class LogIMU380Data: def __init__(self): '''Initialize and create a blob with CSV extension ''' self.name = 'data-' + datetime.datetime.now().strftime( '%Y_%m_%d_%H_%M_%S') + '.csv' self.append_blob_service = AppendBlobService( account_name='navview', account_key= '+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==', protocol='http') self.append_blob_service.create_blob( container_name='data', blob_name=self.name, content_settings=ContentSettings(content_type='text/plain')) self.first_row = 0 self.write_str = '' def log(self, data, odr_setting): '''Buffers and then stores stream based on ODR. Must buffer due to cloud write time. Uses dictionary keys for column titles ''' odr_rates = { 0: 0, 1: 100, 2: 50, 5: 25, 10: 20, 20: 10, 25: 5, 50: 2 } delta_t = 1.0 / odr_rates[odr_setting] if not self.first_row: self.first_row = 1 header = ''.join('{0:s},'.format(key) for key in data) header = header[:-1] header = 'sample,' + header header = header + '\r\n' else: header = '' self.first_row += 1 str = '' for key in data: if key == 'BITstatus' or key == 'GPSITOW' or key == 'counter' or key == 'timeITOW': str += '{0:d},'.format(data[key]) else: str += '{0:3.5f},'.format(data[key]) str = str[:-1] str = '{0:5.2f},'.format(delta_t * (self.first_row - 1)) + str str = str + '\r\n' self.write_str = self.write_str + header + str if (self.first_row % 100 == 0): self.write_to_azure() def write_to_azure(self): '''Appends buffered CSV string to current Azure blob ''' self.append_blob_service.append_blob_from_text('data', self.name, self.write_str) self.write_str = '' def close(self): '''Closes blob ''' self.write_to_azure() self.name = ''
import subprocess import datetime import time from azure.storage.blob import AppendBlobService # Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer append_blob_service = AppendBlobService(account_name='storage_account_name', account_key='storage_account_key') # Creates an append blob for this app. append_blob_service.create_container('distances') append_blob_service.create_blob('distances', 'log.txt') append_blob_service.append_blob_from_text( 'distances', 'log.txt', "Starting: " + datetime.datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n") LatLongDict = {} # Reads the number of jobs from the command line. jobCount = int(sys.argv[1]) # Reads the airport data in to a list for easy access. with open('airports-world.csv') as csvfile: reader = csv.DictReader(csvfile) for row in reader: LatLongDict[row['LocationID']] = [row['Latitude'], row['Longitude']] # Configures the job size based on the job count passed in. jobSize = int(len(LatLongDict) / jobCount) + 1
import datetime import pandas as pd input = pd.read_csv("auto.csv") # input.Symbol[i] input.Series.size is the size all_stock_codes = input.Symbol from nsetools import Nse nse = Nse() outputlist = [] for script_code in all_stock_codes: try: print(script_code) q = nse.get_quote(script_code) outputlist.append(q) print(q) # Write to a list except: print("Unexpected error:", sys.exc_info()[0]) jsonObject = json.dumps(outputlist) from azure.storage.blob import AppendBlobService append_blob_service = AppendBlobService( account_name='aphrostock', account_key= 'cwAlI7P6WfUqijt0jMP+0CCqM54hmoHRofxqdj9PqPamtIfdm9vRUpC+jrpRv/Idma61sSg7NDIvFwkyhMm7KQ==' ) containerName = str(datetime.datetime.now()) append_blob_service.create_blob('stockdata', containerName) append_blob_service.append_blob_from_text('stockdata', containerName, jsonObject)
#from azure.storage.blob import ContentSettings #block_blob_service.create_blob_from_path( # 'myseccontainer', # 'remotesecfiles3.txt', # 'localfiles3.txt', # content_settings=ContentSettings(content_type='text/html') # ) #### To list the blobs in a container, use the list_blobs method. This method returns a generator. #### The following code outputs the name of each blob in a container to the console. #generator = block_blob_service.list_blobs('myseccontainer') #for blob in generator: # print(blob.name) #### The following example demonstrates using get_blob_to_path to download the contents of the myblob blob and store it to the out-sunset.png file. #block_blob_service.get_blob_to_path('myseccontainer', 'remotesecf.txt', 'fromazure-out.txt') #### Finally, to delete a blob, call delete_blob. block_blob_service.delete_blob('myseccontainer', 'remotesecf.txt') #### The example below creates a new append blob and appends some data to it, simulating a simple logging operation. from azure.storage.blob import AppendBlobService append_blob_service = AppendBlobService(myaccount, mykey) #The same containers can hold all types of blobs append_blob_service.create_container('mycontainer') #Append blobs must be created before they are appended to append_blob_service.create_blob('mycontainer', 'myappendblob') append_blob_service.append_blob_from_text('mycontainer', 'myappendblob', u'Sinaq, cumle!') append_blob = append_blob_service.get_blob_to_text('mycontainer', 'myappendblob') print(append_blob)
import storageconfig as cfg from azure.storage.blob import AppendBlobService # Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer append_blob_service = AppendBlobService( account_name=cfg.storage['storage_account_name'], account_key=cfg.storage['storage_account_key']) # Creates an append blob for this app. append_blob_service.create_container(cfg.storage['container_name']) append_blob_service.create_blob( cfg.storage['container_name'], cfg.storage['blob_name']) append_blob_service.append_blob_from_text(cfg.storage['container_name'], cfg.storage['blob_name'], "Starting: " + datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n") LatLongDict = {} # Reads the number of jobs from the command line. jobCount = int(sys.argv[1]) # Reads the airport data in to a list for easy access. with open('airports-world.csv') as csvfile: reader = csv.DictReader(csvfile) for row in reader: LatLongDict[row['LocationID']] = [row['Latitude'], row['Longitude']] # Configures the job size based on the job count passed in.
class FileLoger(): def __init__(self, device_properties): '''Initialize and create a CSV file ''' start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') self.device_properties = device_properties if not self.device_properties: print('No properties found') os._exit(1) self.root_folder = os.path.join(resource.get_executor_path(), r'data') if not os.path.exists(self.root_folder): os.mkdir(self.root_folder) self.output_packets = self.device_properties['userMessages'][ 'outputPackets'] self.log_file_rows = {} self.log_file_names = {} self.log_files_obj = {} self.log_files = {} self.user_file_name = '' # the prefix of log file name. self.msgs_need_to_log = [] self.ws = False # azure app. self.user_id = '' self.file_name = '' self.sas_token = '' self.db_user_access_token = '' # 'http://40.118.233.18:3000/' # TODO: set a host url self.host_url = get_config().ANS_PLATFORM_URL # self.threads = [] # thread of receiver and paser self.exit_thread = False # flag of exit threads self.exit_lock = threading.Lock() # lock of exit_thread self.data_dict = {} # data container self.data_lock = threading.Lock() # lock of data_queue self.device_log_info = None self.ans_platform = AnsPlatformAPI() def start_user_log(self, file_name='', ws=False): ''' start log. return: 0: OK 1: exception that has started logging already. 2: other exception. ''' try: if len(self.log_file_rows) > 0: return 1 # has started logging already. self.ws = ws self.exit_thread = False self.user_file_name = file_name start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') current_path = os.path.join(self.root_folder, start_time) if not os.path.exists(current_path): os.mkdir(current_path) for packet in self.output_packets: # if 1 == packet['save2file']: has_save2file = packet.__contains__('save2file') save2file = 1 if has_save2file: save2file = packet['save2file'] if save2file == 1: self.msgs_need_to_log.append(packet['name']) self.log_file_rows[packet['name']] = 0 if self.user_file_name == '': self.log_file_names[ packet['name']] = packet['name'] + '.csv' else: self.log_file_names[packet['name']] = self.user_file_name + \ '_' + packet['name'] + '.csv' self.log_files[packet['name']] = self.log_file_names[ packet['name']] self.log_files_obj[packet['name']] = open( current_path + '/' + self.log_file_names[packet['name']], 'w') if self.ws: self.get_sas_token() self.data_dict.clear() for i, (k, v) in enumerate(self.log_files.items() ): # k:pack type v:log file name self.data_dict[v] = '' threading.Thread(target=self.upload_azure, args=(k, v)).start() return 0 except Exception as e: print('Exception! File:[{0}], Line:[{1}]. Exception:{2}'.format( __file__, sys._getframe().f_lineno, e)) return 2 def stop_user_log(self): ''' stop log. return: 0: OK 1: exception that driver hasn't started logging files yet. 2: other exception. ''' rev = 0 try: if len(self.log_file_rows) == 0: return 1 # driver hasn't started logging files yet. for i, (k, v) in enumerate(self.log_files_obj.items()): v.close() self.log_file_rows.clear() self.log_file_names.clear() self.log_files_obj.clear() rev = 0 except Exception as e: print(e) rev = 2 if self.ws: time.sleep(1) self.exit_lock.acquire() self.exit_thread = True self.exit_lock.release() self.ws = False return rev def upload_azure(self, packet_type, log_file_name): if self.db_user_access_token == '' or self.sas_token == '': print( "Error: Can not upload log to azure since token is empty! Please check the network." ) print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:'), log_file_name, ' start.') config = get_config() account_name = config.AZURE_STORAGE_ACCOUNT container_name = config.AZURE_STORAGE_DATA_CONTAINER url_name = datetime.datetime.now().strftime( '%Y_%m_%d_%H_%M_%S') + '-' + self.user_id + '-' + log_file_name bcreate_blob_ok = False error_connection = 'ConnectionError' error_authorization = 'AuthenticationFailed' while True: # get data from data_dict. self.data_lock.acquire() text = self.data_dict[log_file_name] self.data_dict[log_file_name] = '' self.data_lock.release() # check if user stop logging data. self.exit_lock.acquire() if self.exit_thread: # check for internet and text if text == '' or (not self.internet_on()): self.exit_lock.release() break else: pass self.exit_lock.release() # let CPU have a break. if text == '': time.sleep(1) continue # create blob on azure if not bcreate_blob_ok: try: self.append_blob_service = AppendBlobService( account_name=account_name, sas_token=self.sas_token, protocol='http') self.append_blob_service.create_blob( container_name=container_name, blob_name=url_name, content_settings=ContentSettings( content_type='text/plain')) bcreate_blob_ok = True threading.Thread(target=self.save_to_db_task, args=(packet_type, log_file_name, url_name)).start() except Exception as e: # print('Exception when create_blob:', type(e), e) if error_connection in str(e): pass elif error_authorization in str(e): self.get_sas_token() self.append_blob_service = AppendBlobService( account_name=account_name, sas_token=self.sas_token, protocol='http') print('Retry to create_blob again...') continue # append blob on azure try: # self.append_blob_service.append_blob_from_text(countainerName, fileName, text, progress_callback=self.upload_callback) self.append_blob_service.append_blob_from_text( container_name, url_name, text) except Exception as e: # print('Exception when append_blob:', type(e), e) if error_connection in str(e): pass elif error_authorization in str(e): self.get_sas_token() self.append_blob_service = AppendBlobService( account_name=account_name, sas_token=self.sas_token, protocol='http') # if append blob failed, do not drop 'text', but push 'text' to data_dict and re-append next time. self.data_lock.acquire() self.data_dict[log_file_name] = text + \ self.data_dict[log_file_name] self.data_lock.release() if bcreate_blob_ok: # if not self.save_to_ans_platform(packet_type, log_file_name): # print('save_to_ans_platform failed.') print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:'), log_file_name, ' done.') def save_to_db_task(self, packet_type, file_name, url_name): if not self.save_to_ans_platform(packet_type, file_name, url_name): print('save_to_ans_platform failed.') def append(self, packet_type, packet): if len(self.log_file_rows) == 0: # if hasn't started logging. return if packet_type in self.msgs_need_to_log: self.log(packet_type, packet) def get_log_file_names(self): return self.log_file_names.copy() def log(self, packet_type, data): ''' Parse the data, read in from the unit, and generate a data file using the json properties file to create a header and specify the precision of the data in the resulting data file. ''' output_packet = next( (x for x in self.output_packets if x['name'] == packet_type), None) fields = [field['name'] for field in output_packet['payload']] '''Write row of CSV file based on data received. Uses dictionary keys for column titles ''' if self.log_file_rows[packet_type] == 0: # Loop through each item in the data dictionary and create a header from the json # properties that correspond to the items in the dictionary labels = '' # for key in data: for i, (k, v) in enumerate(data.items()): '''dataStr = output_packet['payload'][i]['name'] + \ ' [' + \ output_packet['payload'][i]['unit'] + \ ']''' if not fields.__contains__(k): continue data_str = output_packet['payload'][i]['name'] unit_str = output_packet['payload'][i]['unit'] if unit_str == '': labels = labels + '{0:s},'.format(data_str) else: labels = labels + \ '{0:s} ({1:s}),'.format(data_str, unit_str) # Remove the comma at the end of the string and append a new-line character labels = labels[:-1] header = labels + '\n' else: header = '' self.log_file_rows[packet_type] += 1 # Loop through the items in the data dictionary and append to an output string # (with precision based on the data type defined in the json properties file) str = '' for i, (k, v) in enumerate(data.items()): if not fields.__contains__(k): continue output_packet_type = output_packet['payload'][i]['type'] if output_packet['payload'][i].__contains__('scaling'): str += '{0},'.format(v) else: if output_packet_type == 'uint32' or output_packet_type == 'int32' or \ output_packet_type == 'uint16' or output_packet_type == 'int16' or \ output_packet_type == 'uint64' or output_packet_type == 'int64': # integers and unsigned integers str += '{0:d},'.format(v) elif output_packet_type == 'double': # double str += '{0:0.8f},'.format(v) # 15.12 elif output_packet_type == 'float': str += '{0:0.4f},'.format(v) # 12.8 elif output_packet_type == 'uint8': # byte str += '{0:d},'.format(v) elif output_packet_type == 'uchar' or output_packet_type == 'char' or output_packet_type == 'string': # character str += '{:},'.format(v) else: # unknown str += '{0:3.5f},'.format(v) # str = header + str[:-1] + '\n' self.log_files_obj[packet_type].write(str) self.log_files_obj[packet_type].flush() if self.ws: self.data_lock.acquire() self.data_dict[self.log_files[packet_type]] = self.data_dict[ self.log_files[packet_type]] + str self.data_lock.release() def set_info(self, info): self.device_log_info = info pass def set_user_id(self, user_id): self.user_id = user_id if not isinstance(self.user_id, str): self.user_id = str(self.user_id) def set_user_access_token(self, access_token): self.db_user_access_token = access_token def get_sas_token(self): try: self.ans_platform.set_access_token(self.db_user_access_token) self.sas_token = self.ans_platform.get_sas_token() except Exception as e: self.sas_token = '' print('Exception when get_sas_token:', e) def save_to_ans_platform(self, packet_type, file_name, url_name): ''' Upload CSV related information to the database. ''' if not self.device_log_info: return False try: self.device_log_info['fileName'] = file_name self.device_log_info['url'] = url_name self.device_log_info['userId'] = self.user_id self.device_log_info['logInfo']['packetType'] = packet_type data = self.device_log_info # data = { # "type": self.device_log_info['type'], # "model": self.device_log_info['name'], # "fileName": file_name, # "url": file_name, # "userId": self.user_id, # "logInfo": { # "pn": self.device_log_info['pn'], # "sn": self.device_log_info['sn'], # "packetType": packet_type, # "insProperties": json.dumps(self.device_properties) # } # } self.ans_platform.set_access_token(self.db_user_access_token) return self.ans_platform.save_record_log(data) except Exception as e: print('Exception when update db:', e) def internet_on(self): try: url = 'https://navview.blob.core.windows.net/' if sys.version_info[0] > 2: import urllib.request response = urllib.request.urlopen(url, timeout=1) else: import urllib2 response = urllib2.urlopen(url, timeout=1) # print(response.read()) return True except urllib2.URLError as err: return False
class OpenIMULog: def __init__(self, imu, user = False): '''Initialize and create a CSV file ''' self.name = 'data-' + datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + '.csv' if user: self.user = user if self.user['fileName'] == '': self.user['fileName'] = self.name else: self.user['fileName'] += '.csv' self.file = open('data/' + self.user['fileName'], 'w') else: self.file = open('data/' + self.name, 'w') self.first_row = 0 # decode converts out of byte array self.ws = imu.ws self.sn = imu.device_id.split(" ")[0] self.pn = imu.device_id.split(" ")[1] self.device_id = imu.device_id self.odr_setting = imu.odr_setting self.packet_type = imu.packet_type self.imu_properties = imu.imu_properties # Parse the data, read in from the unit, and generate a data file using # the json properties file to create a header and specify the precision # of the data in the resulting data file. def log(self, imu, data): # output_packet = next((x for x in imu.imu_properties['userMessages']['outputPackets'] if x['name'] == imu.packet_type), None) '''Write row of CSV file based on data received. Uses dictionary keys for column titles ''' if not self.first_row: self.first_row = 1 # Loop through each item in the data dictionary and create a header from the json # properties that correspond to the items in the dictionary labels = '' keyIdx = -1 for key in data: keyIdx= keyIdx + 1 '''dataStr = output_packet['payload'][keyIdx]['name'] + \ ' [' + \ output_packet['payload'][keyIdx]['unit'] + \ ']''' dataStr = output_packet['payload'][keyIdx]['name'] labels = labels + '{0:s},'.format(dataStr) # Remove the comma at the end of the string and append a new-line character labels = labels[:-1] header = labels + '\n' else: self.first_row += 1 header = '' # Loop through the items in the data dictionary and append to an output string # (with precision based on the data type defined in the json properties file) str = '' keyIdx = -1 for key in data: keyIdx= keyIdx + 1 outputPcktType = output_packet['payload'][keyIdx]['type'] if outputPcktType == 'uint32' or outputPcktType == 'int32' or \ outputPcktType == 'uint16' or outputPcktType == 'int16' or \ outputPcktType == 'uint64' or outputPcktType == 'int64': # integers and unsigned integers str += '{0:d},'.format(data[key]) elif outputPcktType == 'double': # double str += '{0:15.12f},'.format(data[key]) elif outputPcktType == 'float': # print(3) #key + str(2)) str += '{0:12.8f},'.format(data[key]) elif outputPcktType == 'uint8': # byte str += '{0:d},'.format(data[key]) elif outputPcktType == 'uchar' or outputPcktType == 'char': # character str += '{:},'.format(data[key]) else: # unknown print(0) str += '{0:3.5f},'.format(data[key]) # str = str[:-1] str = str + '\n' self.file.write(header+str) def write_to_azure(self): # check for internet # if not self.internet_on(): # return False # record file to cloud # f = open("data/" + self.name,"r") f = open("data/" + self.user['fileName'], "r") text = f.read() try: self.append_blob_service = AppendBlobService(account_name='navview', account_key='+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==', protocol='http') self.append_blob_service.create_blob(container_name='data', blob_name=self.name, content_settings=ContentSettings(content_type='text/plain')) self.append_blob_service.append_blob_from_text('data',self.name, text) except: # Try again! print('trying to write again due to exception') self.append_blob_service = AppendBlobService(account_name='navview', account_key='+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==', protocol='http') self.append_blob_service.create_blob(container_name='data', blob_name=self.name, content_settings=ContentSettings(content_type='text/plain')) self.append_blob_service.append_blob_from_text('data',self.name, text) # record record to ansplatform self.record_to_ansplatform() def record_to_ansplatform(self): data = { "pn" : self.pn, "sn": self.sn, "fileName" : self.user['fileName'], "url" : self.name, "imuProperties" : json.dumps(self.imu_properties), "sampleRate" : self.odr_setting, "packetType" : self.packet_type, "userId" : self.user['id'] } url = "https://api.aceinna.com/api/datafiles/replaceOrCreate" data_json = json.dumps(data) headers = {'Content-type': 'application/json', 'Authorization' : self.user['access_token'] } response = requests.post(url, data=data_json, headers=headers) response = response.json() # clean up self.name = '' return #ends thread def internet_on(self): try: urllib2.urlopen('https://ans-platform.azurewebsites.net', timeout=1) return True except urllib2.URLError as err: return False def close(self): time.sleep(0.1) if self.ws: self.file.close() threading.Thread(target=self.write_to_azure).start() else: self.file.close()
class AzureBlobStore21(implements(StoreInterface)): def __init__(self, storage_creds, max_retries=10): self.storage_id = storage_creds["name"] self.storage_key = storage_creds["key"] self.bs = BlockBlobService(account_name=self.storage_id, account_key=self.storage_key) self.append_bs = AppendBlobService(account_name=self.storage_id, account_key=self.storage_key) self.max_retries = max_retries self.set_retries(max_retries) # ---- HELPER functions ---- def set_retries(self, count): old_count = self.max_retries self.max_retries = count # bug workaround: standard Retry classes don't retry status=409 (container is being deleted) #import azure.storage.common.retry as retry #self.bs.retry = retry.LinearRetry(backoff=5, max_attempts=count).retry #self.append_bs.retry = retry.LinearRetry(backoff=5, max_attempts=count).retry self.bs.retry = utils.make_retry_func(count) self.append_bs.retry = utils.make_retry_func(count) return old_count # ---- MISC part of interface ---- def get_service_name(self): ''' return the unique name of the storage service''' return self.storage_id def get_retry(self): return self.bs.retry def set_retry(self, value): self.bs.retry = value # ---- CONTAINER interface ---- def does_container_exist(self, container): return self.bs.exists(container) def create_container(self, container): return self.bs.create_container(container) def list_containers(self): containers = self.bs.list_containers() name_list = [contain.name for contain in containers] return name_list def delete_container(self, container): return self.bs.delete_container(container) def get_container_properties(self, container): props = self.bs.get_container_properties(container) return props def get_container_metadata(self, container): md = self.bs.get_container_metadata(container) return md # def set_container_metadata(self, container, md_dict): # return self.bs.set_container_metadata(container, md_dict) # ---- BLOB interface ---- def does_blob_exist(self, container, blob_path): return self.bs.exists(container, blob_path) def create_blob(self, container, blob_path, text, fail_if_exists=False): ifn = "*" if fail_if_exists else None return self.bs.create_blob_from_text(container, blob_path, text, if_none_match=ifn) def create_blob_from_path(self, container, blob_path, source_fn, progress_callback=None): result = self.bs.create_blob_from_path( container, blob_path, source_fn, progress_callback=progress_callback) return result def append_blob(self, container, blob_path, text, append_with_rewrite=False): # create blob if it doesn't exist if not append_with_rewrite: # normal handling if not self.append_bs.exists(container, blob_path): self.append_bs.create_blob(container, blob_path) return self.append_bs.append_blob_from_text( container, blob_path, text) ''' Appends text to a normal blob blob by reading and then rewriting the entire blob. Correctly handles concurrency/race conditions. Recommended for lots of small items (like 10,000 run names). Note: we turn off retries on azure CALL-level so that we can retry on OUR CALL-level. ''' # experimental local retry loop old_retry = self.bs.get_retry() self.bs.set_retry(utils.make_retry_func(0)) succeeded = False for i in range(20): try: if self.bs.does_blob_exist(container, blob_path): # read prev contents blob_text = self.bs.get_blob_text(container, blob_path) # append our text new_text = blob_text + text # write blob, ensuring etag matches (no one updated since above read) self.bs.create_blob(container, blob_path, new_text, if_match=blob.properties.etag) else: # if no previous blob, just try to create it self.bs.create_blob(container, blob_path, text) except BaseException as ex: logger.exception( "Error in _append_blob_with_retries, ex={}".format(ex)) sleep_time = np.random.random() * 4 console.diag( "XT store received an expected azure exception; will backoff for {:.4f} secs [retry #{}]" .format(sleep_time, i + 1)) time.sleep(sleep_time) else: succeeded = True break # restore retry self.bs.set_retry(old_retry) if not succeeded: errors.service_error( "_append_blob_with_rewrite failed (too many retries)") def list_blobs(self, container, path=None, return_names=True, recursive=True): ''' NOTE: the semantics here a tricky if recursive: - return a flat list of all full path names of all files (no directory entries) else: - return a flat list of all files and all directory names (add "/" to end of directory names) if return_names: - return list of names else: - return a list of objects with following properties: .name (file pathname) .properties .content_length (number) .modified_ns (time in ns) The delimiter trick: this is when we set the delimiter arg = "/" to tell azure to return only the blobs in the specified directory - that is, don't return blobs from child directories. In this case, azure returns the effective child directory name, followed by a "/", but not its contents (which we hope is faster). ''' delimiter = None if recursive else "/" # specific Azure path rules for good results if path: if path.startswith("/"): path = path[ 1:] # blob API wants this part of path relative to container # we should only add a "/" if path is a folder path if path.endswith("*"): # we just need to block the addition of "/" path = path[0:-1] elif not path.endswith("/"): path += "/" # best if path ends with "/" blobs = self.bs.list_blobs(container, prefix=path, delimiter=delimiter) if return_names: blobs = [blob.name for blob in blobs] else: blobs = list(blobs) return blobs def delete_blob(self, container, blob_path, snapshot=None): dss = DeleteSnapshot() return self.bs.delete_blob(container, blob_path, delete_snapshots=dss.Include) def get_blob_text(self, container, blob_path): # watch out for 0-length blobs - they trigger an Azure RETRY error text = "" # azure storage bug workaround: avoid RETRY errors for 0-length blob blob = self.bs.get_blob_properties(container, blob_path) if blob.properties.content_length: blob = self.bs.get_blob_to_text(container, blob_path) text = blob.content return text def get_blob_to_path(self, container, blob_path, dest_fn, snapshot=None, progress_callback=None): # azure storage bug workaround: avoid RETRY errors for 0-length blob blob = self.bs.get_blob_properties(container, blob_path) if blob.properties.content_length: result = self.bs.get_blob_to_path( container, blob_path, dest_fn, snapshot=snapshot, progress_callback=progress_callback) text = result.content else: md = blob.metadata if "hdi_isfolder" in md and md["hdi_isfolder"]: # its a directory marker; do NOT create a local file for it text = "" else: # 0-length text file; just write the file outselves text = "" with open(dest_fn, "wt") as outfile: outfile.write(text) return text def get_blob_properties(self, container, blob_path): props = self.bs.get_blob_properties(container, blob_path) return props def get_blob_metadata(self, container, blob_path): return self.bs.get_blob_metadata(container, blob_path) # def set_blob_metadata(self, container, blob_path, md_dict): # return self.bs.set_blob_metadata(container, blob_path, md_dict) def copy_blob(self, source_container, source_blob_path, dest_container, dest_blob_path): source_blob_url = self.bs.make_blob_url(source_container, source_blob_path) self.bs.copy_blob(dest_container, dest_blob_path, source_blob_url) def snapshot_blob(self, container, blob_path): blob = self.bs.snapshot_blob(container, blob_path) #pd = utils.obj_to_dict(blob) return blob
import json import requests import pprint from azure.storage.blob import AppendBlobService from azure.storage.blob import ContentSettings append_blob_service = AppendBlobService( account_name='navview', account_key= '+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==', protocol='http') append_blob_service.create_blob( container_name='data', blob_name="data-2018_05_04_13_13_24.csv", content_settings=ContentSettings(content_type='text/plain')) f = open("data/data-2018_05_04_13_13_24.csv", "r") append_blob_service.append_blob_from_text('data', "data-2018_05_04_13_13_24.csv", f.read())
account_name=cfg.storage['storage_account_name'], account_key=cfg.storage['storage_account_key']) # Reads the start and stop index passed in through SLURM start = int(sys.argv[1]) stop = int(sys.argv[2]) # Creates the blob for this batch. append_blob_service.create_blob( 'distances', str(start) + "-" + str(stop) + '.csv') # Logs the start time append_blob_service.append_blob_from_text( cfg.storage['container_name'], cfg.storage['blob_name'], "Starting " + str(start) + "-" + str( stop) + ":" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n") LatLongDict = {} # radius of earth in miles R = 3959.0 # Reads the airport data in to a list for easy access. with open('airports-world.csv') as csvfile: reader = csv.DictReader(csvfile) for row in reader: LatLongDict[row['LocationID']] = [row['Latitude'], row['Longitude']] # Creates the column names for the distance table
class LogWriter(object): """description of class""" LOG_CONTAINER_NAME = r'log-files' DEBUG_MODE = bool(os.getenv('DEBUG_MODE', False)) # コンストラクタ def __init__(self, name, key, subFolderName=None): super(LogWriter, self).__init__() self._name = name self._key = key self.m_szLogFileName = "" self.m_szSubFolderName = subFolderName self.m_pBlobService = AppendBlobService(name, key) #}def __init__ def _CreateLogFile(self): """ ログファイルを作成する。WriteLog を呼び出す前に実行すること。 """ szRet = "" if (LogWriter.DEBUG_MODE): return ("Debug モードのためスキップします。") try: if (0 == len(self.m_szLogFileName)): szRet = "create_container" bIsExists = self.m_pBlobService.exists( LogWriter.LOG_CONTAINER_NAME) if bIsExists: pass else: self.m_pBlobService.create_container( LogWriter.LOG_CONTAINER_NAME, public_access=PublicAccess.Blob) #ログファイル名の決定 #// 後ろに追加しているが len で 0 と調べているため空文字列 if ((self.m_szSubFolderName is not None) and (0 < len(self.m_szSubFolderName))): #// サブフォルダー名が指定されているときは追加する self.m_szLogFileName += self.m_szSubFolderName + "\\" #}if self.m_szLogFileName += r"{0:%Y-%m-%dT%H-%M-%S.log}".format( datetime.datetime.now()) bIsExists = self.m_pBlobService.exists( LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName) if bIsExists: szRet = "already blob." else: szRet = "create_blob" self.m_pBlobService.create_blob( LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName) szRet = "OK" else: szRet = "Already called." szRet = "OK" #}if except Exception as e: #szRet = "Log exception"; szRet = szRet + "\r\n" + str(e) pass return szRet #}def def WriteLog(self, txt): """ ログファイルにテキストを出力する。末尾に改行コードが追加される。 """ szRet = "" szLogText = r"{0:%Y-%m-%d %H:%M:%S}".format( datetime.datetime.now()) + r" : " + txt + "\r\n" if (LogWriter.DEBUG_MODE): print(szLogText) return ("Debug モードのためスキップしました。") try: #ログファイルの作成 self._CreateLogFile() szRet = "append_blob_from_text" self.m_pBlobService.append_blob_from_text( LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName, szLogText) szRet = "OK" except Exception as e: #szRet = "Log exception"; szRet = szRet + "\r\n" + str(e) #try return szRet #}def def WriteBlob(self, blob_name, value): """ 単一 BLOB ファイルを作成しテキストを保存する。 """ szRet = "" if (LogWriter.DEBUG_MODE): return ("Debug モードのため書き込みをしません。") try: #blob_name = r'sample.txt'; szRet = "BlockBlobService" blob_service = BlockBlobService(self._name, self._key) szRet = "create_container" blob_service.create_container(LogWriter.LOG_CONTAINER_NAME, public_access=PublicAccess.Blob) szRet = "create_blob_from_bytes" #blob_service.create_blob_from_bytes( # log_container_name, # log_blob_name, # b'<center><h1>Hello World!</h1></center>', # content_settings=ContentSettings('text/html') #) if (isinstance(value, str)): szRet = "create_blob_from_text" blob_service.create_blob_from_text( LogWriter.LOG_CONTAINER_NAME, blob_name, value) else: szRet = "create_blob_from_stream" blob_service.create_blob_from_stream( LogWriter.LOG_CONTAINER_NAME, blob_name, io.BytesIO(value)) #}if #szRet = "make_blob_url" #print(blob_service.make_blob_url(log_container_name, log_blob_name)) szRet = "OK" except: print(r"Exception.") #try return szRet #def WriteBlob( blob_name, txt ): def MakeBlobUri(self, blob_name): blob_service = BlockBlobService(self._name, self._key) szRet = blob_service.make_blob_url(LogWriter.LOG_CONTAINER_NAME, blob_name) return (szRet) #}def #}class
import subprocess import datetime import time from azure.storage.blob import AppendBlobService # Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer append_blob_service = AppendBlobService( account_name='storage_account_name', account_key='storage_account_key') # Creates an append blob for this app. append_blob_service.create_container('distances') append_blob_service.create_blob('distances', 'log.txt') append_blob_service.append_blob_from_text('distances', 'log.txt', "Starting: " + datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n") LatLongDict = {} # Reads the number of jobs from the command line. jobCount = int(sys.argv[1]) # Reads the airport data in to a list for easy access. with open('airports-world.csv') as csvfile: reader = csv.DictReader(csvfile) for row in reader: LatLongDict[row['LocationID']] = [row['Latitude'], row['Longitude']] # Configures the job size based on the job count passed in. jobSize = int(len(LatLongDict) / jobCount) + 1
# 'myseccontainer', # 'remotesecfiles3.txt', # 'localfiles3.txt', # content_settings=ContentSettings(content_type='text/html') # ) #### To list the blobs in a container, use the list_blobs method. This method returns a generator. #### The following code outputs the name of each blob in a container to the console. #generator = block_blob_service.list_blobs('myseccontainer') #for blob in generator: # print(blob.name) #### The following example demonstrates using get_blob_to_path to download the contents of the myblob blob and store it to the out-sunset.png file. #block_blob_service.get_blob_to_path('myseccontainer', 'remotesecf.txt', 'fromazure-out.txt') #### Finally, to delete a blob, call delete_blob. block_blob_service.delete_blob('myseccontainer', 'remotesecf.txt') #### The example below creates a new append blob and appends some data to it, simulating a simple logging operation. from azure.storage.blob import AppendBlobService append_blob_service = AppendBlobService(myaccount, mykey) #The same containers can hold all types of blobs append_blob_service.create_container('mycontainer') #Append blobs must be created before they are appended to append_blob_service.create_blob('mycontainer', 'myappendblob') append_blob_service.append_blob_from_text('mycontainer', 'myappendblob', u'Sinaq, cumle!') append_blob = append_blob_service.get_blob_to_text('mycontainer', 'myappendblob') print(append_blob)