def update_status(self, status=0, errorMsg=None):
if self.files is not None:
text = str(status) + '&,&' + datetime.datetime.utcnow().strftime(
'%a, %d %b %Y %H:%M:%S GMT') + '&,&' + '&,&'.join(self.files)
else:
text = str(status) + '&,&' + datetime.datetime.utcnow().strftime(
'%a, %d %b %Y %H:%M:%S GMT')
if errorMsg is not None:
text = str(status) + '&,&' + datetime.datetime.utcnow().strftime(
'%a, %d %b %Y %H:%M:%S GMT') + '&,&' + errorMsg
name = self.folderName + '/status.cvs'
append_blob_service = AppendBlobService(
account_name='navview',
account_key=
'+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==',
protocol='http')
append_blob_service.create_blob(
container_name='data',
blob_name=name,
content_settings=ContentSettings(content_type='text/plain'))
append_blob_service.append_blob_from_text(container_name='data',
blob_name=name,
text=text)
#if __name__ == '__main__':
#DataUpload(dirPath='../../demo_saved_data/2018-07-25-13-01-51').begin_update_files()
def update_files(self, fileName):
try:
filePath = os.path.abspath(
os.path.join(os.path.dirname(__file__),
self.dirPath)) + '/' + fileName
f = open(filePath, 'r')
text = f.read()
f.close()
name = self.folderName + '/' + fileName
append_blob_service = AppendBlobService(
account_name='navview',
account_key=
'+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==',
protocol='http')
append_blob_service.create_blob(
container_name='data',
blob_name=name,
content_settings=ContentSettings(content_type='text/plain'))
append_blob_service.append_blob_from_bytes(
container_name='data',
blob_name=name,
blob=text,
progress_callback=self.processCall)
except Exception as e:
print(e)
def test_add_block_to_append_blob_with_container_sas(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
token = self.bs.generate_container_shared_access_signature(
self.container_name,
permission=ContainerPermissions.READ +
ContainerPermissions.CREATE + ContainerPermissions.ADD,
expiry=datetime.utcnow() + timedelta(hours=1),
)
bs_with_sas = AppendBlobService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
protocol=self.settings.PROTOCOL)
# Act
test_blob_name = "testblobname"
text_blob_content = "test-blob-content"
bs_with_sas.create_blob(self.container_name, blob_name=test_blob_name)
bs_with_sas.append_blob_from_text(self.container_name,
blob_name=test_blob_name,
text=text_blob_content)
blob_content = bs_with_sas.get_blob_to_text(self.container_name,
blob_name=test_blob_name)
# Assert
self.assertEqual(blob_content.content, text_blob_content)
def azure_blob_storage():
try:
# To create a blob service or connection to azure blob storage
block_blob_service = BlockBlobService(account_name='account_name', account_key='account_key')
# To create a container
container_name ='container_name'
block_blob_service.create_container(container_name)
# Set access to a container such as public ,readonly,private
block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container)
# Create a file to upload to azure blob storage
local_file_name ="Test.txt"
path_to_file =os.path.join('local_path', local_file_name)
# To write to the file
local_file = open(full_path_to_file, 'w')
local_file.write("hi peoplee")
local_file.close()
# Upload the created file, use local_file_name for the blob name
block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file)
# To List all the blobs in the container
container_List = block_blob_service.list_blobs(container_name)
for file in container_List:
print("Blob name: "+{file.name})
# Download the blob(s).
download_file_path = os.path.join('local_path', 'local_file_name')
block_blob_service.get_blob_to_path(container_name, local_file_name, download_file_path)
#delete a blob
block_blob_service.delete_blob('container_name', 'blob_name')
#Append to a blob service
append_blob_service = AppendBlobService(account_name='myaccount', account_key='mykey')
# The same containers can hold all types of blobs
append_blob_service.create_container('container_name')
#To append file must exists
append_blob_service.create_blob('container_name', 'append_blob')
#Append to a blob service
append_blob_service.append_blob_from_text('container_name', 'append_blob', 'Hello, world!')
append_blob = append_blob_service.get_blob_to_text('container_name', 'append_blob')
# Clean up resources. This includes the container and the temp files
block_blob_service.delete_container(container_name)
except Exception as e:
def open_stats_blob_json():
append_blob_service = AppendBlobService(STORAGE_ACCT_NAME, BLOB_KEY)
file_contents = "{" + "\"ProcessDate\": \"{}\",".format(current_date)
#file_contents = "{"
try:
append_blob_service.create_blob(BLOB_CONTAINER,
STATS_FILE_NAME,
if_none_match='*')
except:
print("Tried to Create Existing Blob")
append_blob_service.append_blob_from_bytes(BLOB_CONTAINER, STATS_FILE_NAME,
file_contents.encode())
class AppendBlob:
'''Append blob used to append data, each time called append, the content will be append to the blob'''
def __init__(self):
#account = account_name or secretconf["azure"]["storage"][0]["account"]
#key = account_key or secretconf["azure"]["storage"][0]["key"]
connstr = os.getenv(
"AZURE_STORAGE_CONNECTION_STRING",
False) or secretconf["azure"]["storage"][0]["connection_string"]
self.abservice = AppendBlobService(connection_string=connstr)
def create(self, container, blob, metadata=None):
'''Create an empty blob
Args:
container: name of the container
blob: name of the blob, use '/' to create a folder
metadata: meta data (dict object, value must be str) of the text
Returns:
url of blob
'''
self.abservice.create_blob(container,
blob,
metadata=metadata,
content_settings=textcontent,
if_none_match="*")
now = datetime.now()
start = now + timedelta(-1, 0, 0)
expiry = now + timedelta(365, 0, 0)
sastoken = self.abservice.generate_blob_shared_access_signature(
container,
blob,
permission=blobpermission,
expiry=expiry,
start=start)
return self.abservice.make_blob_url(container,
blob,
sas_token=sastoken)
def appendText(self, container, blob, text, metadata=None):
'''Append text to blob'''
self.abservice.append_blob_from_text(container, blob, text)
def CreateLogFile():
""" ログファイルを作成する。WriteLog を呼び出す前に実行すること。 """
szRet = "";
if( DEBUG_MODE ):
return( "Debug モードのためスキップします。" );
try:
szRet = "AppendBlobService";
blob_service = AppendBlobService(
account_name,
account_key
);
szRet = "create_container";
bIsExists = blob_service.exists(
log_container_name
);
if bIsExists:
pass;
else:
blob_service.create_container(
log_container_name,
public_access=PublicAccess.Blob
);
bIsExists = blob_service.exists(
log_container_name,
log_file_name
);
if bIsExists:
szRet = "already blob."
else:
szRet = "create_blob";
blob_service.create_blob(
log_container_name,
log_file_name
);
szRet = "OK";
except:
#szRet = "Log exception";
pass;
return szRet;
def CreateLogFile():
""" ログファイルを作成する。WriteLog を呼び出す前に実行すること。 """
szRet = ""
try:
szRet = "AppendBlobService"
blob_service = AppendBlobService(account_name, account_key)
szRet = "create_container"
bIsExists = blob_service.exists(log_container_name)
if bIsExists:
pass
else:
blob_service.create_container(log_container_name,
public_access=PublicAccess.Blob)
bIsExists = blob_service.exists(log_container_name, log_file_name)
if bIsExists:
szRet = "already blob."
else:
szRet = "create_blob"
blob_service.create_blob(log_container_name, log_file_name)
szRet = "OK"
except:
#szRet = "Log exception";
pass
return szRet
class FileLoger():
def __init__(self, device_properties):
'''Initialize and create a CSV file
'''
start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
self.device_properties = device_properties
if not self.device_properties:
print('No properties found')
os._exit(1)
self.root_folder = os.path.join(resource.get_executor_path(), r'data')
if not os.path.exists(self.root_folder):
os.mkdir(self.root_folder)
self.output_packets = self.device_properties['userMessages'][
'outputPackets']
self.log_file_rows = {}
self.log_file_names = {}
self.log_files_obj = {}
self.log_files = {}
self.user_file_name = '' # the prefix of log file name.
self.msgs_need_to_log = []
self.ws = False
# azure app.
self.user_id = ''
self.file_name = ''
self.sas_token = ''
self.db_user_access_token = ''
# 'http://40.118.233.18:3000/' # TODO: set a host url
self.host_url = get_config().ANS_PLATFORM_URL
#
self.threads = [] # thread of receiver and paser
self.exit_thread = False # flag of exit threads
self.exit_lock = threading.Lock() # lock of exit_thread
self.data_dict = {} # data container
self.data_lock = threading.Lock() # lock of data_queue
self.device_log_info = None
self.ans_platform = AnsPlatformAPI()
def start_user_log(self, file_name='', ws=False):
'''
start log.
return:
0: OK
1: exception that has started logging already.
2: other exception.
'''
try:
if len(self.log_file_rows) > 0:
return 1 # has started logging already.
self.ws = ws
self.exit_thread = False
self.user_file_name = file_name
start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
current_path = os.path.join(self.root_folder, start_time)
if not os.path.exists(current_path):
os.mkdir(current_path)
for packet in self.output_packets:
# if 1 == packet['save2file']:
has_save2file = packet.__contains__('save2file')
save2file = 1
if has_save2file:
save2file = packet['save2file']
if save2file == 1:
self.msgs_need_to_log.append(packet['name'])
self.log_file_rows[packet['name']] = 0
if self.user_file_name == '':
self.log_file_names[
packet['name']] = packet['name'] + '.csv'
else:
self.log_file_names[packet['name']] = self.user_file_name + \
'_' + packet['name'] + '.csv'
self.log_files[packet['name']] = self.log_file_names[
packet['name']]
self.log_files_obj[packet['name']] = open(
current_path + '/' + self.log_file_names[packet['name']],
'w')
if self.ws:
self.get_sas_token()
self.data_dict.clear()
for i, (k, v) in enumerate(self.log_files.items()
): # k:pack type v:log file name
self.data_dict[v] = ''
threading.Thread(target=self.upload_azure,
args=(k, v)).start()
return 0
except Exception as e:
print('Exception! File:[{0}], Line:[{1}]. Exception:{2}'.format(
__file__,
sys._getframe().f_lineno, e))
return 2
def stop_user_log(self):
'''
stop log.
return:
0: OK
1: exception that driver hasn't started logging files yet.
2: other exception.
'''
rev = 0
try:
if len(self.log_file_rows) == 0:
return 1 # driver hasn't started logging files yet.
for i, (k, v) in enumerate(self.log_files_obj.items()):
v.close()
self.log_file_rows.clear()
self.log_file_names.clear()
self.log_files_obj.clear()
rev = 0
except Exception as e:
print(e)
rev = 2
if self.ws:
time.sleep(1)
self.exit_lock.acquire()
self.exit_thread = True
self.exit_lock.release()
self.ws = False
return rev
def upload_azure(self, packet_type, log_file_name):
if self.db_user_access_token == '' or self.sas_token == '':
print(
"Error: Can not upload log to azure since token is empty! Please check the network."
)
print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:'),
log_file_name, ' start.')
config = get_config()
account_name = config.AZURE_STORAGE_ACCOUNT
container_name = config.AZURE_STORAGE_DATA_CONTAINER
url_name = datetime.datetime.now().strftime(
'%Y_%m_%d_%H_%M_%S') + '-' + self.user_id + '-' + log_file_name
bcreate_blob_ok = False
error_connection = 'ConnectionError'
error_authorization = 'AuthenticationFailed'
while True:
# get data from data_dict.
self.data_lock.acquire()
text = self.data_dict[log_file_name]
self.data_dict[log_file_name] = ''
self.data_lock.release()
# check if user stop logging data.
self.exit_lock.acquire()
if self.exit_thread:
# check for internet and text
if text == '' or (not self.internet_on()):
self.exit_lock.release()
break
else:
pass
self.exit_lock.release()
# let CPU have a break.
if text == '':
time.sleep(1)
continue
# create blob on azure
if not bcreate_blob_ok:
try:
self.append_blob_service = AppendBlobService(
account_name=account_name,
sas_token=self.sas_token,
protocol='http')
self.append_blob_service.create_blob(
container_name=container_name,
blob_name=url_name,
content_settings=ContentSettings(
content_type='text/plain'))
bcreate_blob_ok = True
threading.Thread(target=self.save_to_db_task,
args=(packet_type, log_file_name,
url_name)).start()
except Exception as e:
# print('Exception when create_blob:', type(e), e)
if error_connection in str(e):
pass
elif error_authorization in str(e):
self.get_sas_token()
self.append_blob_service = AppendBlobService(
account_name=account_name,
sas_token=self.sas_token,
protocol='http')
print('Retry to create_blob again...')
continue
# append blob on azure
try:
# self.append_blob_service.append_blob_from_text(countainerName, fileName, text, progress_callback=self.upload_callback)
self.append_blob_service.append_blob_from_text(
container_name, url_name, text)
except Exception as e:
# print('Exception when append_blob:', type(e), e)
if error_connection in str(e):
pass
elif error_authorization in str(e):
self.get_sas_token()
self.append_blob_service = AppendBlobService(
account_name=account_name,
sas_token=self.sas_token,
protocol='http')
# if append blob failed, do not drop 'text', but push 'text' to data_dict and re-append next time.
self.data_lock.acquire()
self.data_dict[log_file_name] = text + \
self.data_dict[log_file_name]
self.data_lock.release()
if bcreate_blob_ok:
# if not self.save_to_ans_platform(packet_type, log_file_name):
# print('save_to_ans_platform failed.')
print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:'),
log_file_name, ' done.')
def save_to_db_task(self, packet_type, file_name, url_name):
if not self.save_to_ans_platform(packet_type, file_name, url_name):
print('save_to_ans_platform failed.')
def append(self, packet_type, packet):
if len(self.log_file_rows) == 0: # if hasn't started logging.
return
if packet_type in self.msgs_need_to_log:
self.log(packet_type, packet)
def get_log_file_names(self):
return self.log_file_names.copy()
def log(self, packet_type, data):
''' Parse the data, read in from the unit, and generate a data file using
the json properties file to create a header and specify the precision
of the data in the resulting data file.
'''
output_packet = next(
(x for x in self.output_packets if x['name'] == packet_type), None)
fields = [field['name'] for field in output_packet['payload']]
'''Write row of CSV file based on data received. Uses dictionary keys for column titles
'''
if self.log_file_rows[packet_type] == 0:
# Loop through each item in the data dictionary and create a header from the json
# properties that correspond to the items in the dictionary
labels = ''
# for key in data:
for i, (k, v) in enumerate(data.items()):
'''dataStr = output_packet['payload'][i]['name'] + \
' [' + \
output_packet['payload'][i]['unit'] + \
']'''
if not fields.__contains__(k):
continue
data_str = output_packet['payload'][i]['name']
unit_str = output_packet['payload'][i]['unit']
if unit_str == '':
labels = labels + '{0:s},'.format(data_str)
else:
labels = labels + \
'{0:s} ({1:s}),'.format(data_str, unit_str)
# Remove the comma at the end of the string and append a new-line character
labels = labels[:-1]
header = labels + '\n'
else:
header = ''
self.log_file_rows[packet_type] += 1
# Loop through the items in the data dictionary and append to an output string
# (with precision based on the data type defined in the json properties file)
str = ''
for i, (k, v) in enumerate(data.items()):
if not fields.__contains__(k):
continue
output_packet_type = output_packet['payload'][i]['type']
if output_packet['payload'][i].__contains__('scaling'):
str += '{0},'.format(v)
else:
if output_packet_type == 'uint32' or output_packet_type == 'int32' or \
output_packet_type == 'uint16' or output_packet_type == 'int16' or \
output_packet_type == 'uint64' or output_packet_type == 'int64':
# integers and unsigned integers
str += '{0:d},'.format(v)
elif output_packet_type == 'double':
# double
str += '{0:0.8f},'.format(v) # 15.12
elif output_packet_type == 'float':
str += '{0:0.4f},'.format(v) # 12.8
elif output_packet_type == 'uint8':
# byte
str += '{0:d},'.format(v)
elif output_packet_type == 'uchar' or output_packet_type == 'char' or output_packet_type == 'string':
# character
str += '{:},'.format(v)
else:
# unknown
str += '{0:3.5f},'.format(v)
#
str = header + str[:-1] + '\n'
self.log_files_obj[packet_type].write(str)
self.log_files_obj[packet_type].flush()
if self.ws:
self.data_lock.acquire()
self.data_dict[self.log_files[packet_type]] = self.data_dict[
self.log_files[packet_type]] + str
self.data_lock.release()
def set_info(self, info):
self.device_log_info = info
pass
def set_user_id(self, user_id):
self.user_id = user_id
if not isinstance(self.user_id, str):
self.user_id = str(self.user_id)
def set_user_access_token(self, access_token):
self.db_user_access_token = access_token
def get_sas_token(self):
try:
self.ans_platform.set_access_token(self.db_user_access_token)
self.sas_token = self.ans_platform.get_sas_token()
except Exception as e:
self.sas_token = ''
print('Exception when get_sas_token:', e)
def save_to_ans_platform(self, packet_type, file_name, url_name):
''' Upload CSV related information to the database.
'''
if not self.device_log_info:
return False
try:
self.device_log_info['fileName'] = file_name
self.device_log_info['url'] = url_name
self.device_log_info['userId'] = self.user_id
self.device_log_info['logInfo']['packetType'] = packet_type
data = self.device_log_info
# data = {
# "type": self.device_log_info['type'],
# "model": self.device_log_info['name'],
# "fileName": file_name,
# "url": file_name,
# "userId": self.user_id,
# "logInfo": {
# "pn": self.device_log_info['pn'],
# "sn": self.device_log_info['sn'],
# "packetType": packet_type,
# "insProperties": json.dumps(self.device_properties)
# }
# }
self.ans_platform.set_access_token(self.db_user_access_token)
return self.ans_platform.save_record_log(data)
except Exception as e:
print('Exception when update db:', e)
def internet_on(self):
try:
url = 'https://navview.blob.core.windows.net/'
if sys.version_info[0] > 2:
import urllib.request
response = urllib.request.urlopen(url, timeout=1)
else:
import urllib2
response = urllib2.urlopen(url, timeout=1)
# print(response.read())
return True
except urllib2.URLError as err:
return False
#from azure.storage.blob import ContentSettings
#block_blob_service.create_blob_from_path(
# 'myseccontainer',
# 'remotesecfiles3.txt',
# 'localfiles3.txt',
# content_settings=ContentSettings(content_type='text/html')
# )
#### To list the blobs in a container, use the list_blobs method. This method returns a generator.
#### The following code outputs the name of each blob in a container to the console.
#generator = block_blob_service.list_blobs('myseccontainer')
#for blob in generator:
# print(blob.name)
#### The following example demonstrates using get_blob_to_path to download the contents of the myblob blob and store it to the out-sunset.png file.
#block_blob_service.get_blob_to_path('myseccontainer', 'remotesecf.txt', 'fromazure-out.txt')
#### Finally, to delete a blob, call delete_blob.
block_blob_service.delete_blob('myseccontainer', 'remotesecf.txt')
#### The example below creates a new append blob and appends some data to it, simulating a simple logging operation.
from azure.storage.blob import AppendBlobService
append_blob_service = AppendBlobService(myaccount, mykey)
#The same containers can hold all types of blobs
append_blob_service.create_container('mycontainer')
#Append blobs must be created before they are appended to
append_blob_service.create_blob('mycontainer', 'myappendblob')
append_blob_service.append_blob_from_text('mycontainer', 'myappendblob', u'Sinaq, cumle!')
append_blob = append_blob_service.get_blob_to_text('mycontainer', 'myappendblob')
print(append_blob)
class LogIMU380Data:
def __init__(self, imu, user):
'''Initialize and create a CSV file
'''
self.name = 'data-' + datetime.datetime.now().strftime(
'%Y_%m_%d_%H_%M_%S') + '.csv'
self.file = open('data/' + self.name, 'w')
self.first_row = 0
# decode converts out of byte array
self.sn = imu.device_id.split(" ")[0]
self.pn = imu.device_id.split(" ")[1]
self.device_id = imu.device_id
self.odr_setting = imu.odr_setting
self.packet_type = imu.packet_type
odr_rates = {0: 'Quiet', 1: '100Hz', 2: '50Hz', 4: '25Hz'}
self.sample_rate = odr_rates[self.odr_setting]
def log(self, data, odr_setting):
'''Write row of CSV file based on data received. Uses dictionary keys for column titles
'''
if not self.first_row:
self.first_row = 1
labels = ''.join('{0:s},'.format(key) for key in data)
labels = labels[:-1]
header = labels + '\n'
else:
self.first_row += 1
header = ''
str = ''
for key in data:
if key == 'BITstatus' or key == 'GPSITOW' or key == 'counter' or key == 'timeITOW':
str += '{0:d},'.format(data[key])
else:
str += '{0:3.5f},'.format(data[key])
str = str[:-1]
str = str + '\n'
self.file.write(header + str)
def write_to_azure(self):
# check for internet
# if not self.internet_on():
# return False
# record file to cloud
self.append_blob_service = AppendBlobService(
account_name='navview',
account_key=
'+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==',
protocol='http')
self.append_blob_service.create_blob(
container_name='data',
blob_name=self.name,
content_settings=ContentSettings(content_type='text/plain'))
f = open("data/" + self.name, "r")
self.append_blob_service.append_blob_from_text('data', self.name,
f.read())
# TODO: check if success
# record record to ansplatform
self.record_to_ansplatform()
def record_to_ansplatform(self):
data = {
"pn": self.pn,
"sn": self.sn,
"fileName": self.user['fileName'],
"url": self.name,
"imuProperties": json.dumps(self.imu_properties),
"sampleRate": self.sample_rate,
"packetType": self.packet_type,
"userId": self.user['id']
}
url = "https://ans-platform.azurewebsites.net/api/datafiles/replaceOrCreate"
data_json = json.dumps(data)
headers = {
'Content-type': 'application/json',
'Authorization': self.user['access_token']
}
response = requests.post(url, data=data_json, headers=headers)
response = response.json()
print(response)
# clean up
self.file.close()
self.name = ''
return #ends thread
def internet_on(self):
try:
urllib2.urlopen('https://ans-platform.azurewebsites.net',
timeout=1)
return True
except urllib2.URLError as err:
return False
def close(self):
time.sleep(0.1)
threading.Thread(target=self.write_to_azure).start()
# generator = block_blob_service.list_blobs('mycontainer')
# for blob in generator:
# print(blob.name)
# block_blob_service.get_blob_to_path('mycontainer', 'myblockblob', 'out-sunset.png')
# block_blob_service.delete_blob('mycontainer', 'myblockblob')
from azure.storage.blob import AppendBlobService
append_blob_service = AppendBlobService(account_name='bobur', account_key='6e60FZapOXAmUbFBw0SpE1lHRP3RkXOMYRaalWmRBoz4+xI5tvjaJzxXuYyt+yfWxjPXpz5X3PmyIFiQmSkjbw==')
# The same containers can hold all types of blobs
append_blob_service.create_container('myappendcontainer')
# Append blobs must be created before they are appended to
append_blob_service.create_blob('myappendcontainer', 'myappendblob')
append_blob_service.append_blob_from_text('myappendcontainer', 'myappendblob', u'Hello, world!')
append_blob = append_blob_service.get_blob_to_text('myappendcontainer', 'myappendblob')
print append_blob.content
class OpenIMULog:
def __init__(self, imu, user = False):
'''Initialize and create a CSV file
'''
self.name = 'data-' + datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + '.csv'
if user:
self.user = user
if self.user['fileName'] == '':
self.user['fileName'] = self.name
else:
self.user['fileName'] += '.csv'
self.file = open('data/' + self.user['fileName'], 'w')
else:
self.file = open('data/' + self.name, 'w')
self.first_row = 0
# decode converts out of byte array
self.ws = imu.ws
self.sn = imu.device_id.split(" ")[0]
self.pn = imu.device_id.split(" ")[1]
self.device_id = imu.device_id
self.odr_setting = imu.odr_setting
self.packet_type = imu.packet_type
self.imu_properties = imu.imu_properties
# Parse the data, read in from the unit, and generate a data file using
# the json properties file to create a header and specify the precision
# of the data in the resulting data file.
def log(self, imu, data):
#
output_packet = next((x for x in imu.imu_properties['userMessages']['outputPackets'] if x['name'] == imu.packet_type), None)
'''Write row of CSV file based on data received. Uses dictionary keys for column titles
'''
if not self.first_row:
self.first_row = 1
# Loop through each item in the data dictionary and create a header from the json
# properties that correspond to the items in the dictionary
labels = ''
keyIdx = -1
for key in data:
keyIdx= keyIdx + 1
'''dataStr = output_packet['payload'][keyIdx]['name'] + \
' [' + \
output_packet['payload'][keyIdx]['unit'] + \
']'''
dataStr = output_packet['payload'][keyIdx]['name']
labels = labels + '{0:s},'.format(dataStr)
# Remove the comma at the end of the string and append a new-line character
labels = labels[:-1]
header = labels + '\n'
else:
self.first_row += 1
header = ''
# Loop through the items in the data dictionary and append to an output string
# (with precision based on the data type defined in the json properties file)
str = ''
keyIdx = -1
for key in data:
keyIdx= keyIdx + 1
outputPcktType = output_packet['payload'][keyIdx]['type']
if outputPcktType == 'uint32' or outputPcktType == 'int32' or \
outputPcktType == 'uint16' or outputPcktType == 'int16' or \
outputPcktType == 'uint64' or outputPcktType == 'int64':
# integers and unsigned integers
str += '{0:d},'.format(data[key])
elif outputPcktType == 'double':
# double
str += '{0:15.12f},'.format(data[key])
elif outputPcktType == 'float':
# print(3) #key + str(2))
str += '{0:12.8f},'.format(data[key])
elif outputPcktType == 'uint8':
# byte
str += '{0:d},'.format(data[key])
elif outputPcktType == 'uchar' or outputPcktType == 'char':
# character
str += '{:},'.format(data[key])
else:
# unknown
print(0)
str += '{0:3.5f},'.format(data[key])
#
str = str[:-1]
str = str + '\n'
self.file.write(header+str)
def write_to_azure(self):
# check for internet
# if not self.internet_on():
# return False
# record file to cloud
# f = open("data/" + self.name,"r")
f = open("data/" + self.user['fileName'], "r")
text = f.read()
try:
self.append_blob_service = AppendBlobService(account_name='navview', account_key='+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==', protocol='http')
self.append_blob_service.create_blob(container_name='data', blob_name=self.name, content_settings=ContentSettings(content_type='text/plain'))
self.append_blob_service.append_blob_from_text('data',self.name, text)
except:
# Try again!
print('trying to write again due to exception')
self.append_blob_service = AppendBlobService(account_name='navview', account_key='+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==', protocol='http')
self.append_blob_service.create_blob(container_name='data', blob_name=self.name, content_settings=ContentSettings(content_type='text/plain'))
self.append_blob_service.append_blob_from_text('data',self.name, text)
# record record to ansplatform
self.record_to_ansplatform()
def record_to_ansplatform(self):
data = { "pn" : self.pn, "sn": self.sn, "fileName" : self.user['fileName'], "url" : self.name, "imuProperties" : json.dumps(self.imu_properties),
"sampleRate" : self.odr_setting, "packetType" : self.packet_type, "userId" : self.user['id'] }
url = "https://api.aceinna.com/api/datafiles/replaceOrCreate"
data_json = json.dumps(data)
headers = {'Content-type': 'application/json', 'Authorization' : self.user['access_token'] }
response = requests.post(url, data=data_json, headers=headers)
response = response.json()
# clean up
self.name = ''
return #ends thread
def internet_on(self):
try:
urllib2.urlopen('https://ans-platform.azurewebsites.net', timeout=1)
return True
except urllib2.URLError as err:
return False
def close(self):
time.sleep(0.1)
if self.ws:
self.file.close()
threading.Thread(target=self.write_to_azure).start()
else:
self.file.close()
import sys
import subprocess
import datetime
import time
import storageconfig as cfg
from azure.storage.blob import AppendBlobService
# Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer
append_blob_service = AppendBlobService(
account_name=cfg.storage['storage_account_name'],
account_key=cfg.storage['storage_account_key'])
# Creates an append blob for this app.
append_blob_service.create_container(cfg.storage['container_name'])
append_blob_service.create_blob(
cfg.storage['container_name'], cfg.storage['blob_name'])
append_blob_service.append_blob_from_text(cfg.storage['container_name'],
cfg.storage['blob_name'], "Starting: " +
datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n")
LatLongDict = {}
# Reads the number of jobs from the command line.
jobCount = int(sys.argv[1])
# Reads the airport data in to a list for easy access.
with open('airports-world.csv') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
LatLongDict[row['LocationID']] = [row['Latitude'], row['Longitude']]
from math import sin, cos, sqrt, atan2, radians
import datetime
import time
from azure.storage.blob import AppendBlobService
# Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer
append_blob_service = AppendBlobService(account_name='storage_account_name',
account_key='storage_account_key')
# Reads the start and stop index passed in through SLURM
start = int(sys.argv[1])
stop = int(sys.argv[2])
#Creates the blob for this batch.
append_blob_service.create_blob('distances',
str(start) + "-" + str(stop) + '.csv')
#Logs the start time
append_blob_service.append_blob_from_text(
'distances', 'log.txt', "Starting " + str(start) + "-" + str(stop) + ":" +
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n")
LatLongDict = {}
# radius of earth in miles
R = 3959.0
# Reads the airport data in to a list for easy access.
with open('airports-world.csv') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
def main():
logging.basicConfig(level=logging.DEBUG)
with open(TASKDATA) as taskdata_file:
taskdata = json.loads(taskdata_file.read())
github = GithubAdapter(taskdata)
gh_commit = github.get_commit()
config = github.get_config()
blob_service = AppendBlobService(
account_name=taskdata["storage_account_name"],
account_key=taskdata["storage_account_key"])
queue_service = QueueService(
connection_string=taskdata["queue_connection_string"])
loop = asyncio.get_event_loop()
ctx = Context(loop=loop,
config=config,
blob_service=blob_service,
queue_service=queue_service,
taskdata=taskdata)
blob_service.create_container("logs",
fail_on_exist=False,
public_access=PublicAccess.Blob)
blob_service.create_blob("logs",
ctx.pid,
content_settings=ContentSettings(
content_type="text/plain; charset=utf-8"))
gh_commit.create_status(
"pending",
target_url=blob_service.make_blob_url("logs", ctx.pid),
description="Build started",
context=config.get("configuration_name",
"configuration%s" % taskdata["config_num"]))
os.makedirs(REPOSDIR, exist_ok=True)
# Check if we're the only process who updates the git cache on SMB share.
# Otherwise skip updating.
if not os.path.exists(LOCKFILENAME):
lock = open(LOCKFILENAME, "w")
lock.close()
update_git_cache(ctx)
os.unlink(LOCKFILENAME)
if os.path.exists(SRCDIR):
shutil.rmtree(SRCDIR)
os.makedirs(os.path.join(SRCDIR, "build/conf"))
with open(os.path.join(SRCDIR, "build/conf/auto.conf"), "a") as localconf:
localconf.write("\n%s\n" % config.get("localconf", ""))
localconf.write(AUTOCONFIG)
repos = get_repos(config)
repos.append((repodirname(taskdata["gh"]["repository"]["clone_url"]),
taskdata["gh"]["repository"]["clone_url"], None, None))
for reponame, repourl, reporef, _ in repos:
refrepopath = os.path.join(REPOSDIR, reponame)
run(ctx,
["git", "clone", "--reference", refrepopath, repourl, reponame],
cwd=SRCDIR)
if reporef:
LOG.info("Checkout %s to %s" % (reponame, reporef))
run(ctx, ["git", "checkout", "%s" % reporef],
cwd=os.path.join(SRCDIR, reponame))
# Do checkout
if taskdata["gh"]["type"] == "pull_request":
LOG.info("Add remote repo %s" % taskdata["gh"]["clone_url"])
run(ctx, [
"git", "remote", "add", "contributor", taskdata["gh"]["clone_url"]
],
cwd=os.path.join(
SRCDIR,
repodirname(taskdata["gh"]["repository"]["clone_url"])))
LOG.info("Fetch contributor's repo")
run(ctx, ["git", "fetch", "contributor"],
cwd=os.path.join(
SRCDIR,
repodirname(taskdata["gh"]["repository"]["clone_url"])))
LOG.info("Checkout %s to %s" % (repodirname(
taskdata["gh"]["repository"]["clone_url"]), taskdata["gh"]["sha"]))
run(ctx, ["git", "checkout", taskdata["gh"]["sha"]],
cwd=os.path.join(
SRCDIR, repodirname(taskdata["gh"]["repository"]["clone_url"])))
# Fetch sstate if any
if os.path.exists(get_sstate_archive_path(ctx)):
with tarfile.open(name=get_sstate_archive_path(ctx),
mode="r:gz") as sstate_tar:
sstate_tar.extractall(path=SRCDIR)
addlayers = []
for dep in config["dependencies"]:
repodir = repodirname(dep["url"])
layers = dep.get("layers", None)
if layers:
addlayers.extend([
"bitbake-layers add-layer ../%s/%s" % (repodir, layer)
for layer in layers
])
else:
addlayers.append("bitbake-layers add-layer ../%s" % repodir)
addlayers.append("bitbake-layers add-layer ../%s" %
repodirname(taskdata["gh"]["repository"]["clone_url"]))
run_script(ctx,
BUILDSCRIPT % ("\n".join(addlayers), config["bitbake_target"]),
cwd=SRCDIR)
save_sstate(ctx)
# Github auth token has expired by now most probably => renew
github = GithubAdapter(taskdata)
gh_commit = github.get_commit()
gh_commit.create_status(
"success",
target_url=blob_service.make_blob_url("logs", ctx.pid),
description="Target has been built successfully",
context=config.get("configuration_name",
"configuration%s" % taskdata["config_num"]))
loop.close()
# TODO: copy cloud-init log files to share
taskdata["build_result"] = "success"
queue_service.put_message(
"buildresults",
base64.b64encode(json.dumps(taskdata).encode("utf")).decode("utf"))
class RoverLogApp(rover_application_base.RoverApplicationBase):
def __init__(self, user=False):
'''Initialize and create a CSV file
'''
if user and list(user.keys())[0] == 'startLog':
self.username = user['startLog']['username']
self.userId = user['startLog']['id']
self.userFilename = user['startLog']['fileName']
self.userAccessToken = user['startLog']['access_token']
self.start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
if not self.load_configuration():
os._exit(1)
if not os.path.exists('data/'):
os.mkdir('data/')
self.output_packets = self.rover_properties['userMessages'][
'outputPackets']
self.first_row = {}
self.log_file_names = {}
self.log_files = {}
if user:
with open('tempLogFiles.json', 'r') as outfile:
data = json.load(outfile)
self.log_file_names = data
if not user:
with open('tempLogFiles.json', 'w') as outfile:
json.dump({}, outfile)
try:
for packet in self.output_packets:
self.first_row[packet['name']] = 0
self.log_file_names[packet['name']] = packet[
'name'] + '-' + self.start_time + '.csv'
self.log_files[packet['name']] = open(
'data/' + self.log_file_names[packet['name']],
'w') # just log Compact Navigation Message
entry = {
packet['name']: self.log_file_names[packet['name']]
}
with open('tempLogFiles.json') as f:
data = json.load(f)
data.update(entry)
with open('tempLogFiles.json', 'w') as f:
json.dump(data, f)
except:
pass
if user and list(user.keys())[0] == 'startLog':
self.savetoAnsPlatform()
if user and list(user.keys())[0] == 'stopLog':
time.sleep(10)
self.close()
# os.remove("tempLogFiles.json")
def on_reinit(self):
print("RoverLogApp.on_reinit()")
pass
# Is it necessary to create a new file to log when replug serial connector?
# self.start_time = datetime.datetime.now().strftime('%Y%m%d_%H_%M_%S')
# try:
# for packet in self.output_packets:
# self.first_row[packet['name']] = 0
# self.log_file_names[packet['name']] = packet['name'] +'-' + self.start_time + '.csv'
# self.log_files[packet['name']] = open('data/' + self.log_file_names[packet['name']], 'w')# just log Compact Navigation Message
# except:
# pass
def on_find_active_rover(self):
print("RoverLogApp.on_find_active_rover()")
def on_message(self, *args):
packet_type = args[0]
self.data = args[1]
is_var_len_frame = args[2]
if is_var_len_frame:
self.log_var_len(self.data, packet_type)
else:
self.log(self.data, packet_type)
def on_exit(self):
pass
def load_configuration(self):
'''
load properties from 'rover.json'
returns: True when load successfully.
False when load failed.
'''
try:
with open('setting/rover.json') as json_data:
self.rover_properties = json.load(json_data)
return True
# except (ValueError, KeyError, TypeError) as error:
except Exception as e:
print(e)
return False
def log(self, data, packet_type):
''' Parse the data, read in from the unit, and generate a data file using
the json properties file to create a header and specify the precision
of the data in the resulting data file.
'''
if not self.rover_properties:
return
output_packet = next(
(x for x in self.rover_properties['userMessages']['outputPackets']
if x['name'] == packet_type), None)
'''Write row of CSV file based on data received. Uses dictionary keys for column titles
'''
if not self.first_row[packet_type]:
self.first_row[packet_type] = 1
# Loop through each item in the data dictionary and create a header from the json
# properties that correspond to the items in the dictionary
labels = ''
keyIdx = -1
for key in data:
keyIdx = keyIdx + 1
'''dataStr = output_packet['payload'][keyIdx]['name'] + \
' [' + \
output_packet['payload'][keyIdx]['unit'] + \
']'''
dataStr = output_packet['payload'][keyIdx]['name']
unitStr = output_packet['payload'][keyIdx]['unit']
if unitStr == '':
labels = labels + '{0:s},'.format(dataStr)
else:
labels = labels + '{0:s}({1:s}),'.format(dataStr, unitStr)
# Remove the comma at the end of the string and append a new-line character
labels = labels[:-1]
header = labels + '\n'
else:
self.first_row[packet_type] += 1
header = ''
# Loop through the items in the data dictionary and append to an output string
# (with precision based on the data type defined in the json properties file)
str = ''
keyIdx = -1
for key in data:
keyIdx = keyIdx + 1
outputPcktType = output_packet['payload'][keyIdx]['type']
if outputPcktType == 'uint32' or outputPcktType == 'int32' or \
outputPcktType == 'uint16' or outputPcktType == 'int16' or \
outputPcktType == 'uint64' or outputPcktType == 'int64':
# integers and unsigned integers
str += '{0:d},'.format(data[key])
elif outputPcktType == 'double':
# double
str += '{0:15.12f},'.format(data[key])
elif outputPcktType == 'float':
# print(3) #key + str(2))
str += '{0:12.8f},'.format(data[key])
elif outputPcktType == 'uint8':
# byte
str += '{0:d},'.format(data[key])
elif outputPcktType == 'uchar' or outputPcktType == 'char':
# character
str += '{:},'.format(data[key])
else:
# unknown
str += '{0:3.5f},'.format(data[key])
#
str = str[:-1]
str = str + '\n'
self.log_files[packet_type].write(header + str)
def log_var_len(self, data, packet_type):
''' Parse the data, read in from the unit, and generate a data file using
the json properties file to create a header and specify the precision
of the data in the resulting data file.
'''
if not self.rover_properties:
return
output_packet = next(
(x for x in self.rover_properties['userMessages']['outputPackets']
if x['name'] == packet_type), None)
'''Write row of CSV file based on data received. Uses dictionary keys for column titles
'''
if not self.first_row[packet_type]:
self.first_row[packet_type] = 1
# Loop through each item in the data dictionary and create a header from the json
# properties that correspond to the items in the dictionary
labels = ''
for value in output_packet['payload']:
dataStr = value['name']
unitStr = value['unit']
if unitStr == '':
labels = labels + '{0:s},'.format(dataStr)
else:
labels = labels + '{0:s}({1:s}),'.format(dataStr, unitStr)
# Remove the comma at the end of the string and append a new-line character
labels = labels[:-1]
header = labels + '\n'
else:
self.first_row[packet_type] += 1
header = ''
# Loop through the items in the data dictionary and append to an output string
# (with precision based on the data type defined in the json properties file)
str = ''
const_str = ''
var_str = ''
var_fileld_tpyes = []
var_fileld_num = len(
output_packet['payload']) - output_packet['var_num']['field_idx']
const_fileld_num = len(output_packet['payload']) - var_fileld_num
for idx, value in enumerate(output_packet['payload']):
if idx >= const_fileld_num:
var_fileld_tpyes.append(value['type'])
for idx, key in enumerate(data):
if idx < const_fileld_num:
outputPcktType = output_packet['payload'][idx]['type']
if outputPcktType == 'uint32' or outputPcktType == 'int32' or \
outputPcktType == 'uint16' or outputPcktType == 'int16' or \
outputPcktType == 'uint64' or outputPcktType == 'int64':
# integers and unsigned integers
const_str += '{0:d},'.format(list(key.values())[0])
elif outputPcktType == 'double':
# double
const_str += '{0:15.12f},'.format(list(key.values())[0])
elif outputPcktType == 'float':
# print(3) #key + str(2))
const_str += '{0:12.8f},'.format(list(key.values())[0])
elif outputPcktType == 'uint8':
# byte
const_str += '{0:d},'.format(list(key.values())[0])
elif outputPcktType == 'uchar' or outputPcktType == 'char':
# character
const_str += '{:},'.format(list(key.values())[0])
else:
# unknown
const_str += '{0:3.5f},'.format(key.values()[0])
else:
idx_key = -1
for k, v in key.items():
idx_key += 1
outputPcktType = var_fileld_tpyes[idx_key]
if outputPcktType == 'uint32' or outputPcktType == 'int32' or \
outputPcktType == 'uint16' or outputPcktType == 'int16' or \
outputPcktType == 'uint64' or outputPcktType == 'int64':
# integers and unsigned integers
var_str += '{0:d},'.format(v)
elif outputPcktType == 'double':
# double
var_str += '{0:15.12f},'.format(v)
elif outputPcktType == 'float':
# print(3) #key + str(2))
var_str += '{0:12.8f},'.format(v)
elif outputPcktType == 'uint8':
# byte
var_str += '{0:d},'.format(v)
elif outputPcktType == 'uchar' or outputPcktType == 'char':
# character
var_str += '{:},'.format(v)
else:
# unknown
var_str += '{0:3.5f},'.format(v)
str = const_str + var_str
str = str[:-1]
str = str + '\n'
self.log_files[packet_type].write(header + str)
header = ''
str = ''
var_str = ''
''' Upload CSV's to Azure container.
'''
def uploadtoAzure(self, fileDisplayName):
# f = open("data/" + self.user['fileName'], "r")
f = open("data/" + fileDisplayName, "r")
text = f.read()
account_key = '+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg=='
try:
self.azureStorage('navview', account_key, 'data', fileDisplayName,
text)
except:
# Try again!
self.azureStorage('navview', account_key, 'data', fileDisplayName,
text)
''' Trigger Database upload
'''
# self.savetoAnsPlatform()
def azureStorage(self, accountName, accountkey, countainerName, fileName,
text):
self.append_blob_service = AppendBlobService(account_name=accountName,
account_key=accountkey,
protocol='http')
self.append_blob_service.create_blob(
container_name=countainerName,
blob_name=fileName,
content_settings=ContentSettings(content_type='text/plain'))
self.append_blob_service.append_blob_from_text(countainerName,
fileName, text)
''' Upload CSV related information to the database.
'''
def savetoAnsPlatform(self):
for files in self.log_file_names:
fileDisplayName = files + "-" + self.userFilename + ".csv"
data = {
"pn": '1.0.0',
"sn": 'rtk',
"fileName": fileDisplayName,
"url": self.log_file_names[files],
"imuProperties": json.dumps(self.rover_properties),
"sampleRate": '100',
"packetType": files,
"userId": self.userId
}
url = "https://api.aceinna.com/api/datafiles/replaceOrCreate"
data_json = json.dumps(data)
headers = {
'Content-type': 'application/json',
'Authorization': self.userAccessToken
}
response = requests.post(url, data=data_json, headers=headers)
response = response.json()
# def close(self,fileName,storedFile):
# time.sleep(0.1)
# # if self.ws:
# storedFile.close()
# threading.Thread(target=self.uploadtoAzure(fileName)).start()
# # else:
# # self.file.close()
# # print('close')
# # try:
# # for packet in self.output_packets:
# # self.log_files[packet['name']].close()
# # threading.Thread(target=self.write_to_azurelog_files[packet['name']]).start()
# # except:
# # pass
def close(self):
for files in self.log_file_names:
self.uploadtoAzure(self.log_file_names[files])
class LogIMU380Data:
def __init__(self):
'''Initialize and create a blob with CSV extension
'''
self.name = 'data-' + datetime.datetime.now().strftime(
'%Y_%m_%d_%H_%M_%S') + '.csv'
self.append_blob_service = AppendBlobService(
account_name='navview',
account_key=
'+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==',
protocol='http')
self.append_blob_service.create_blob(
container_name='data',
blob_name=self.name,
content_settings=ContentSettings(content_type='text/plain'))
self.first_row = 0
self.write_str = ''
def log(self, data, odr_setting):
'''Buffers and then stores stream based on ODR. Must buffer due to cloud write time.
Uses dictionary keys for column titles
'''
odr_rates = {
0: 0,
1: 100,
2: 50,
5: 25,
10: 20,
20: 10,
25: 5,
50: 2
}
delta_t = 1.0 / odr_rates[odr_setting]
if not self.first_row:
self.first_row = 1
header = ''.join('{0:s},'.format(key) for key in data)
header = header[:-1]
header = 'sample,' + header
header = header + '\r\n'
else:
header = ''
self.first_row += 1
str = ''
for key in data:
if key == 'BITstatus' or key == 'GPSITOW' or key == 'counter' or key == 'timeITOW':
str += '{0:d},'.format(data[key])
else:
str += '{0:3.5f},'.format(data[key])
str = str[:-1]
str = '{0:5.2f},'.format(delta_t * (self.first_row - 1)) + str
str = str + '\r\n'
self.write_str = self.write_str + header + str
if (self.first_row % 100 == 0):
self.write_to_azure()
def write_to_azure(self):
'''Appends buffered CSV string to current Azure blob
'''
self.append_blob_service.append_blob_from_text('data', self.name,
self.write_str)
self.write_str = ''
def close(self):
'''Closes blob
'''
self.write_to_azure()
self.name = ''
class Results(object):
"""
Handles interacting with encrypted results in blob storage.
"""
def __init__(self, logger, redisHost, redisPort):
"""
Initializes a new instance of the JobStatus class.
:param logger logger: The logger instance to use for logging
:param str redis_host: Redis host where the Redis Q is running
:param int redis_port: Redis port where the Redis Q is running
"""
self.logger = logger
self.config = Config()
self.redis_host = redisHost
self.redis_port = redisPort
# create an instance of AESCipher to use for encryption
aesHelper = AESHelper(self.config)
self.aescipher = aesHelper.create_aescipher_from_config()
if (self.init_storage_services() is False):
raise Exception(
"Errors occurred instantiating results storage service.")
def init_storage_services(self):
"""
Initializes the storage service clients using values from config.py.
:return: True on success. False on failure.
:rtype: boolean
"""
try:
# creates instance of BlockBlobService and AppendBlobService to use for completed results storage
self.storage_service = BlockBlobService(
account_name=self.config.storage_account_name,
sas_token=self.config.results_container_sas_token)
self.append_storage_service = AppendBlobService(
account_name=self.config.storage_account_name,
sas_token=self.config.results_container_sas_token)
self.storage_service.create_container(
self.config.results_container_name)
# creates instances of Azure QueueService
self.job_status_queue_service = QueueService(
account_name=self.config.storage_account_name,
sas_token=self.config.job_status_queue_sas_token)
self.job_status_queue_service.encode_function = models.QueueMessageFormat.noencode
self.results_queue_service = QueueService(
account_name=self.config.storage_account_name,
sas_token=self.config.results_queue_sas_token)
self.results_queue_service.create_queue(
self.config.results_container_name)
self.results_queue_service.encode_function = models.QueueMessageFormat.noencode
# creates instance of Redis client to use for job status storage
pool = redis.ConnectionPool(host=self.redis_host,
port=self.redis_port)
self.storage_service_cache = redis.Redis(connection_pool=pool)
return True
except Exception as ex:
self.log_exception(ex, self.init_storage_services.__name__)
return False
def log_exception(self, exception, functionName):
"""
Logs an exception to the logger instance for this class.
:param Exception exception: The exception thrown.
:param str functionName: Name of the function where the exception occurred.
"""
self.logger.debug("Exception occurred in: " + functionName)
self.logger.debug(type(exception))
self.logger.debug(exception)
def write_result(self, result):
"""
Encrypts and writes result to queue
:param str result: The result to write to queue
:return: True on success. False on failure.
:rtype: boolean
"""
try:
# encrypt the encoded result and then encode it
encryptedResult = base64.b64encode(self.aescipher.encrypt(result))
# put the encoded result into the azure queue for future consolidation
self.results_queue_service.put_message(
self.config.results_queue_name, encryptedResult)
return True
except Exception as ex:
self.log_exception(ex, self.write_result.__name__)
return False
def count_consolidated_results(self):
"""
Returns a count of results that were consolidated.
"return: int count: Total count of results that were consolidated.
"""
try:
consolidatedResults = self.storage_service_cache.get(
self.config.results_consolidated_count_redis_key)
return consolidatedResults
except Exception as ex:
self.log_exception(ex, self.count_consolidated_results.__name__)
return False
except Exception as ex:
self.log_exception(
ex, self.consolidate_results.__name__ +
" - Error consolidating result blob.")
def consolidate_results(self):
"""
Consolidates all individual result files into single result file in storage. Blobs are deleted once they
are added to the consolidated file.
"return: int count: Total count of results consolidated in result file.
"""
try:
# ensure the consolidated append blob exists
if not self.append_storage_service.exists(
self.config.results_container_name,
blob_name=self.config.results_consolidated_file):
self.append_storage_service.create_blob(
self.config.results_container_name,
self.config.results_consolidated_file)
result_messages = []
with io.BytesIO() as consolidated_result:
while len(result_messages
) < self.config.result_consolidation_size:
messages = self.results_queue_service.get_messages(
self.config.results_queue_name,
min(self.config.result_consolidation_size, 32))
# If the queue is empty, stop and consolidate
if not messages:
break
# add the message to the memory stream
for msg in messages:
consolidated_result.write(msg.content + "\n")
result_messages.append(msg)
# append the results to the consolidated file
consolidated_result.seek(0)
self.append_storage_service.append_blob_from_stream(
self.config.results_container_name,
self.config.results_consolidated_file, consolidated_result)
# remove all of the messages from the queue
num_of_consolidated_results = len(result_messages)
for msg in result_messages:
self.results_queue_service.delete_message(
self.config.results_queue_name, msg.id, msg.pop_receipt)
self.storage_service_cache.incrby(
self.config.results_consolidated_count_redis_key,
num_of_consolidated_results)
# write the count of results we consolidated out to queue to provide status
self.job_status_queue_service.put_message(
self.config.job_status_queue_name,
str(num_of_consolidated_results) + " results consolidated.")
return len(result_messages)
except Exception as ex:
self.log_exception(ex, self.consolidate_results.__name__)
return len(result_messages)
def get_total_jobs_consolidated_status(self):
"""
Write out the the current state of the workload; the percentage of jobs that are completed and consolidated
"return: float status: percentage of completed jobs
"""
# log out total job status
total_scheduled_jobs = self.storage_service_cache.get(
self.config.scheduled_jobs_count_redis_key)
total_consolidated_results = self.storage_service_cache.get(
self.config.results_consolidated_count_redis_key)
if total_consolidated_results is None:
total_consolidated_results = "0"
status_message = "Total: " + total_consolidated_results + "/" + total_scheduled_jobs + " jobs have been successfully processed and consolidated."
self.logger.info(status_message)
self.job_status_queue_service.put_message(
self.config.job_status_queue_name, status_message)
return float(total_consolidated_results) / int(total_scheduled_jobs)
import datetime
import pandas as pd
input = pd.read_csv("auto.csv")
# input.Symbol[i] input.Series.size is the size
all_stock_codes = input.Symbol
from nsetools import Nse
nse = Nse()
outputlist = []
for script_code in all_stock_codes:
try:
print(script_code)
q = nse.get_quote(script_code)
outputlist.append(q)
print(q)
# Write to a list
except:
print("Unexpected error:",
sys.exc_info()[0])
jsonObject = json.dumps(outputlist)
from azure.storage.blob import AppendBlobService
append_blob_service = AppendBlobService(
account_name='aphrostock',
account_key=
'cwAlI7P6WfUqijt0jMP+0CCqM54hmoHRofxqdj9PqPamtIfdm9vRUpC+jrpRv/Idma61sSg7NDIvFwkyhMm7KQ=='
)
containerName = str(datetime.datetime.now())
append_blob_service.create_blob('stockdata', containerName)
append_blob_service.append_blob_from_text('stockdata', containerName,
jsonObject)
class FileStorage(object):
def __init__(self, container_name):
self.account_name = 'moremmrparsingstorage'
self.account_key = 'UxvGtW/HsW83YcYWBa2IF3TCkr5R3zy4F72KxWy/POvzHPZVJwshZTihElfwHBGuoR2lfgGKcHdAh858IhqbjA=='
self.container_name = container_name
self.blob_service = None
self.append_blob_service = None
def connect_blob_service(self, reconnect=False):
if type(self.blob_service).__name__ != 'BlockBlobService' or reconnect:
try:
self.blob_service = BlockBlobService(account_name=self.account_name,
account_key=self.account_key)
self.blob_service.set_container_acl(self.container_name, public_access=PublicAccess.Container)
except Exception as ex:
print('Exception in connection to blob service: {0}'.format(ex))
return False
return True
def connect_append_blob_service(self, reconnect=False):
if type(self.append_blob_service).__name__ != 'AppendBlobService' or reconnect:
try:
self.append_blob_service = AppendBlobService(account_name=self.account_name,
account_key=self.account_key)
except Exception as ex:
print('Exception in connection to append blob service: {0}'.format(ex))
return False
return True
def get_file(self, blob_name, result_file_path):
blob_service_connected = self.connect_blob_service()
if not blob_service_connected:
return False
else:
try:
self.blob_service.get_blob_to_path(container_name=self.container_name,
blob_name=blob_name,
file_path=result_file_path)
except Exception as ex:
if ex.error_code != 'BlobNotFound':
print('Getting file from blob exception: {0}:'.format(ex))
return False
return True
def put_file(self, blob_name, local_file_path, remove_local_file_after=False):
blob_service_connected = self.connect_blob_service()
if not blob_service_connected:
return False
else:
try:
self.blob_service.create_blob_from_path(container_name=self.container_name,
blob_name=blob_name,
file_path=local_file_path)
except Exception as ex:
print('Putting file to blob exception: {0}:'.format(ex))
return False
if remove_local_file_after:
try:
os.remove(local_file_path)
except OSError:
pass
return True
def append_from_file(self, blob_name, local_file_path, remove_local_file_after=False):
blob_service_connected = self.connect_append_blob_service()
if not blob_service_connected:
return False
else:
try:
self.append_blob_service.append_blob_from_path(container_name=self.container_name,
blob_name=blob_name,
file_path=local_file_path)
except Exception as ex:
print('Append from file in blob exception: {0}:'.format(ex))
return False
if remove_local_file_after:
try:
os.remove(local_file_path)
except OSError:
pass
return True
def create_appended_blob(self, blob_name):
blob_service_connected = self.connect_append_blob_service()
if not blob_service_connected:
return False
else:
try:
self.append_blob_service.create_blob(container_name=self.container_name,
blob_name=blob_name)
except Exception as ex:
print('Creating appended file in blob exception: {0}:'.format(ex))
return False
return True
def exists(self, blob_name):
blob_service_connected = self.connect_blob_service()
exists = False
if blob_service_connected:
try:
exists = self.blob_service.exists(container_name=self.container_name, blob_name=blob_name)
except Exception as ex:
print('Checking file existence in blob exception: {0}:'.format(ex))
return exists
def delete_blob(self, blob_name):
blob_service_connected = self.connect_blob_service()
if blob_service_connected:
try:
self.blob_service.delete_blob(container_name=self.container_name,
blob_name=blob_name)
except Exception as ex:
print('Deleting blob exception: {0}:'.format(ex))
return False
return True
def put_array_to_blob(self, blob_name, array=[], compression=True):
res = False
local_file = '/tmp/{0}_{1}'.format(str(uuid.uuid4()), blob_name)
try:
df = pd.DataFrame(array)
except Exception:
df = pd.DataFrame()
if df.shape[0] > 0:
df.to_csv(path_or_buf=local_file,
header=False,
index=False,
compression=(None, 'zip')[compression])
res = self.put_file(blob_name, local_file, True)
return res
def get_blob_as_df(self, blob_name, names=[]):
df = pd.DataFrame()
local_file = '/tmp/{0}_{1}'.format(str(uuid.uuid4()), blob_name)
if self.get_file(blob_name, local_file):
try:
df = pd.read_csv(filepath_or_buffer=local_file, header=None, names=names)
except Exception:
df = pd.DataFrame()
try:
os.remove(local_file)
except OSError:
pass
return df
def append_df_to_blob(self, blob_name, df):
local_file = '/tmp/{0}_{1}'.format(str(uuid.uuid4()), blob_name)
df.to_csv(path_or_buf=local_file, header=None, index=False)
if not self.exists(blob_name):
self.create_appended_blob(blob_name)
self.append_from_file(blob_name, local_file, True)
return True
def list_blobs(self, num_results=5000):
blobs = []
blob_service_connected = self.connect_blob_service()
if blob_service_connected:
try:
blobs = self.blob_service.list_blobs(self.container_name,
num_results=num_results)
except Exception as ex:
print('List blobs exception: {0}'.format(ex))
return blobs
import json
import requests
import pprint
from azure.storage.blob import AppendBlobService
from azure.storage.blob import ContentSettings
append_blob_service = AppendBlobService(
account_name='navview',
account_key=
'+roYuNmQbtLvq2Tn227ELmb6s1hzavh0qVQwhLORkUpM0DN7gxFc4j+DF/rEla1EsTN2goHEA1J92moOM/lfxg==',
protocol='http')
append_blob_service.create_blob(
container_name='data',
blob_name="data-2018_05_04_13_13_24.csv",
content_settings=ContentSettings(content_type='text/plain'))
f = open("data/data-2018_05_04_13_13_24.csv", "r")
append_blob_service.append_blob_from_text('data',
"data-2018_05_04_13_13_24.csv",
f.read())
class AzureBlobStore21(implements(StoreInterface)):
def __init__(self, storage_creds, max_retries=10):
self.storage_id = storage_creds["name"]
self.storage_key = storage_creds["key"]
self.bs = BlockBlobService(account_name=self.storage_id,
account_key=self.storage_key)
self.append_bs = AppendBlobService(account_name=self.storage_id,
account_key=self.storage_key)
self.max_retries = max_retries
self.set_retries(max_retries)
# ---- HELPER functions ----
def set_retries(self, count):
old_count = self.max_retries
self.max_retries = count
# bug workaround: standard Retry classes don't retry status=409 (container is being deleted)
#import azure.storage.common.retry as retry
#self.bs.retry = retry.LinearRetry(backoff=5, max_attempts=count).retry
#self.append_bs.retry = retry.LinearRetry(backoff=5, max_attempts=count).retry
self.bs.retry = utils.make_retry_func(count)
self.append_bs.retry = utils.make_retry_func(count)
return old_count
# ---- MISC part of interface ----
def get_service_name(self):
''' return the unique name of the storage service'''
return self.storage_id
def get_retry(self):
return self.bs.retry
def set_retry(self, value):
self.bs.retry = value
# ---- CONTAINER interface ----
def does_container_exist(self, container):
return self.bs.exists(container)
def create_container(self, container):
return self.bs.create_container(container)
def list_containers(self):
containers = self.bs.list_containers()
name_list = [contain.name for contain in containers]
return name_list
def delete_container(self, container):
return self.bs.delete_container(container)
def get_container_properties(self, container):
props = self.bs.get_container_properties(container)
return props
def get_container_metadata(self, container):
md = self.bs.get_container_metadata(container)
return md
# def set_container_metadata(self, container, md_dict):
# return self.bs.set_container_metadata(container, md_dict)
# ---- BLOB interface ----
def does_blob_exist(self, container, blob_path):
return self.bs.exists(container, blob_path)
def create_blob(self, container, blob_path, text, fail_if_exists=False):
ifn = "*" if fail_if_exists else None
return self.bs.create_blob_from_text(container,
blob_path,
text,
if_none_match=ifn)
def create_blob_from_path(self,
container,
blob_path,
source_fn,
progress_callback=None):
result = self.bs.create_blob_from_path(
container,
blob_path,
source_fn,
progress_callback=progress_callback)
return result
def append_blob(self,
container,
blob_path,
text,
append_with_rewrite=False):
# create blob if it doesn't exist
if not append_with_rewrite:
# normal handling
if not self.append_bs.exists(container, blob_path):
self.append_bs.create_blob(container, blob_path)
return self.append_bs.append_blob_from_text(
container, blob_path, text)
'''
Appends text to a normal blob blob by reading and then rewriting the entire blob.
Correctly handles concurrency/race conditions.
Recommended for lots of small items (like 10,000 run names).
Note: we turn off retries on azure CALL-level so that we can retry on
OUR CALL-level.
'''
# experimental local retry loop
old_retry = self.bs.get_retry()
self.bs.set_retry(utils.make_retry_func(0))
succeeded = False
for i in range(20):
try:
if self.bs.does_blob_exist(container, blob_path):
# read prev contents
blob_text = self.bs.get_blob_text(container, blob_path)
# append our text
new_text = blob_text + text
# write blob, ensuring etag matches (no one updated since above read)
self.bs.create_blob(container,
blob_path,
new_text,
if_match=blob.properties.etag)
else:
# if no previous blob, just try to create it
self.bs.create_blob(container, blob_path, text)
except BaseException as ex:
logger.exception(
"Error in _append_blob_with_retries, ex={}".format(ex))
sleep_time = np.random.random() * 4
console.diag(
"XT store received an expected azure exception; will backoff for {:.4f} secs [retry #{}]"
.format(sleep_time, i + 1))
time.sleep(sleep_time)
else:
succeeded = True
break
# restore retry
self.bs.set_retry(old_retry)
if not succeeded:
errors.service_error(
"_append_blob_with_rewrite failed (too many retries)")
def list_blobs(self,
container,
path=None,
return_names=True,
recursive=True):
'''
NOTE: the semantics here a tricky
if recursive:
- return a flat list of all full path names of all files (no directory entries)
else:
- return a flat list of all files and all directory names (add "/" to end of directory names)
if return_names:
- return list of names
else:
- return a list of objects with following properties:
.name (file pathname)
.properties
.content_length (number)
.modified_ns (time in ns)
The delimiter trick: this is when we set the delimiter arg = "/" to tell azure to return only the blobs
in the specified directory - that is, don't return blobs from child directories. In this case, azure
returns the effective child directory name, followed by a "/", but not its contents (which we hope is faster).
'''
delimiter = None if recursive else "/"
# specific Azure path rules for good results
if path:
if path.startswith("/"):
path = path[
1:] # blob API wants this part of path relative to container
# we should only add a "/" if path is a folder path
if path.endswith("*"):
# we just need to block the addition of "/"
path = path[0:-1]
elif not path.endswith("/"):
path += "/" # best if path ends with "/"
blobs = self.bs.list_blobs(container, prefix=path, delimiter=delimiter)
if return_names:
blobs = [blob.name for blob in blobs]
else:
blobs = list(blobs)
return blobs
def delete_blob(self, container, blob_path, snapshot=None):
dss = DeleteSnapshot()
return self.bs.delete_blob(container,
blob_path,
delete_snapshots=dss.Include)
def get_blob_text(self, container, blob_path):
# watch out for 0-length blobs - they trigger an Azure RETRY error
text = ""
# azure storage bug workaround: avoid RETRY errors for 0-length blob
blob = self.bs.get_blob_properties(container, blob_path)
if blob.properties.content_length:
blob = self.bs.get_blob_to_text(container, blob_path)
text = blob.content
return text
def get_blob_to_path(self,
container,
blob_path,
dest_fn,
snapshot=None,
progress_callback=None):
# azure storage bug workaround: avoid RETRY errors for 0-length blob
blob = self.bs.get_blob_properties(container, blob_path)
if blob.properties.content_length:
result = self.bs.get_blob_to_path(
container,
blob_path,
dest_fn,
snapshot=snapshot,
progress_callback=progress_callback)
text = result.content
else:
md = blob.metadata
if "hdi_isfolder" in md and md["hdi_isfolder"]:
# its a directory marker; do NOT create a local file for it
text = ""
else:
# 0-length text file; just write the file outselves
text = ""
with open(dest_fn, "wt") as outfile:
outfile.write(text)
return text
def get_blob_properties(self, container, blob_path):
props = self.bs.get_blob_properties(container, blob_path)
return props
def get_blob_metadata(self, container, blob_path):
return self.bs.get_blob_metadata(container, blob_path)
# def set_blob_metadata(self, container, blob_path, md_dict):
# return self.bs.set_blob_metadata(container, blob_path, md_dict)
def copy_blob(self, source_container, source_blob_path, dest_container,
dest_blob_path):
source_blob_url = self.bs.make_blob_url(source_container,
source_blob_path)
self.bs.copy_blob(dest_container, dest_blob_path, source_blob_url)
def snapshot_blob(self, container, blob_path):
blob = self.bs.snapshot_blob(container, blob_path)
#pd = utils.obj_to_dict(blob)
return blob
class LogWriter(object):
"""description of class"""
LOG_CONTAINER_NAME = r'log-files'
DEBUG_MODE = bool(os.getenv('DEBUG_MODE', False))
# コンストラクタ
def __init__(self, name, key, subFolderName=None):
super(LogWriter, self).__init__()
self._name = name
self._key = key
self.m_szLogFileName = ""
self.m_szSubFolderName = subFolderName
self.m_pBlobService = AppendBlobService(name, key)
#}def __init__
def _CreateLogFile(self):
""" ログファイルを作成する。WriteLog を呼び出す前に実行すること。 """
szRet = ""
if (LogWriter.DEBUG_MODE):
return ("Debug モードのためスキップします。")
try:
if (0 == len(self.m_szLogFileName)):
szRet = "create_container"
bIsExists = self.m_pBlobService.exists(
LogWriter.LOG_CONTAINER_NAME)
if bIsExists:
pass
else:
self.m_pBlobService.create_container(
LogWriter.LOG_CONTAINER_NAME,
public_access=PublicAccess.Blob)
#ログファイル名の決定
#// 後ろに追加しているが len で 0 と調べているため空文字列
if ((self.m_szSubFolderName is not None)
and (0 < len(self.m_szSubFolderName))):
#// サブフォルダー名が指定されているときは追加する
self.m_szLogFileName += self.m_szSubFolderName + "\\"
#}if
self.m_szLogFileName += r"{0:%Y-%m-%dT%H-%M-%S.log}".format(
datetime.datetime.now())
bIsExists = self.m_pBlobService.exists(
LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName)
if bIsExists:
szRet = "already blob."
else:
szRet = "create_blob"
self.m_pBlobService.create_blob(
LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName)
szRet = "OK"
else:
szRet = "Already called."
szRet = "OK"
#}if
except Exception as e:
#szRet = "Log exception";
szRet = szRet + "\r\n" + str(e)
pass
return szRet
#}def
def WriteLog(self, txt):
""" ログファイルにテキストを出力する。末尾に改行コードが追加される。 """
szRet = ""
szLogText = r"{0:%Y-%m-%d %H:%M:%S}".format(
datetime.datetime.now()) + r" : " + txt + "\r\n"
if (LogWriter.DEBUG_MODE):
print(szLogText)
return ("Debug モードのためスキップしました。")
try:
#ログファイルの作成
self._CreateLogFile()
szRet = "append_blob_from_text"
self.m_pBlobService.append_blob_from_text(
LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName, szLogText)
szRet = "OK"
except Exception as e:
#szRet = "Log exception";
szRet = szRet + "\r\n" + str(e)
#try
return szRet
#}def
def WriteBlob(self, blob_name, value):
""" 単一 BLOB ファイルを作成しテキストを保存する。 """
szRet = ""
if (LogWriter.DEBUG_MODE):
return ("Debug モードのため書き込みをしません。")
try:
#blob_name = r'sample.txt';
szRet = "BlockBlobService"
blob_service = BlockBlobService(self._name, self._key)
szRet = "create_container"
blob_service.create_container(LogWriter.LOG_CONTAINER_NAME,
public_access=PublicAccess.Blob)
szRet = "create_blob_from_bytes"
#blob_service.create_blob_from_bytes(
# log_container_name,
# log_blob_name,
# b'<center><h1>Hello World!</h1></center>',
# content_settings=ContentSettings('text/html')
#)
if (isinstance(value, str)):
szRet = "create_blob_from_text"
blob_service.create_blob_from_text(
LogWriter.LOG_CONTAINER_NAME, blob_name, value)
else:
szRet = "create_blob_from_stream"
blob_service.create_blob_from_stream(
LogWriter.LOG_CONTAINER_NAME, blob_name, io.BytesIO(value))
#}if
#szRet = "make_blob_url"
#print(blob_service.make_blob_url(log_container_name, log_blob_name))
szRet = "OK"
except:
print(r"Exception.")
#try
return szRet
#def WriteBlob( blob_name, txt ):
def MakeBlobUri(self, blob_name):
blob_service = BlockBlobService(self._name, self._key)
szRet = blob_service.make_blob_url(LogWriter.LOG_CONTAINER_NAME,
blob_name)
return (szRet)
#}def
#}class
import storageconfig as cfg
from azure.storage.blob import AppendBlobService
# Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer
append_blob_service = AppendBlobService(
account_name=cfg.storage['storage_account_name'],
account_key=cfg.storage['storage_account_key'])
# Reads the start and stop index passed in through SLURM
start = int(sys.argv[1])
stop = int(sys.argv[2])
# Creates the blob for this batch.
append_blob_service.create_blob(
'distances', str(start) + "-" + str(stop) + '.csv')
# Logs the start time
append_blob_service.append_blob_from_text(
cfg.storage['container_name'],
cfg.storage['blob_name'], "Starting " + str(start) + "-" + str(
stop) + ":" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n")
LatLongDict = {}
# radius of earth in miles
R = 3959.0
# Reads the airport data in to a list for easy access.
with open('airports-world.csv') as csvfile:
class FileLoger():
def __init__(self):
'''Initialize and create a CSV file
'''
start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
self.rover_properties = utility.load_configuration(os.path.join('setting', 'rover.json'))
if not self.rover_properties:
os._exit(1)
if not os.path.exists('data/'):
os.mkdir('data/')
self.output_packets = self.rover_properties['userMessages']['outputPackets']
self.log_file_rows = {}
self.log_file_names = {}
self.log_files_obj = {}
self.log_files = {}
self.user_file_name = '' # the prefix of log file name.
self.msgs_need_to_log = []
self.ws = False
# azure app.
self.user_id = ''
self.file_name = ''
self.sas_token = ''
self.db_user_access_token = ''
self.host_url = self.rover_properties['userConfiguration']['hostURL']
#
self.threads = [] # thread of receiver and paser
self.exit_thread = False # flag of exit threads
self.exit_lock = threading.Lock() # lock of exit_thread
self.data_dict = {} # data container
self.data_lock = threading.Lock() # lock of data_queue
def start_user_log(self, file_name='', ws=False):
'''
start log.
return:
0: OK
1: exception that has started logging already.
2: other exception.
'''
try:
if len(self.log_file_rows) > 0:
return 1 # has started logging already.
self.ws = ws
self.exit_thread = False
self.user_file_name = file_name
start_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
for packet in self.output_packets:
if 1 == packet['save2file']:
self.msgs_need_to_log.append(packet['name'])
self.log_file_rows[packet['name']] = 0
if self.user_file_name == '':
self.log_file_names[packet['name']] = packet['name'] +'-' + start_time + '.csv'
else:
self.log_file_names[packet['name']] = self.user_file_name + '_' + packet['name'] +'-' + start_time + '.csv'
self.log_files[packet['name']] = self.log_file_names[packet['name']]
self.log_files_obj[packet['name']] = open('data/' + self.log_file_names[packet['name']], 'w')
if self.ws:
self.get_sas_token()
self.data_dict.clear()
for i, (k, v) in enumerate(self.log_files.items()): # k:pack type v:log file name
self.data_dict[v]=''
threading.Thread(target=self.upload_azure, args=(k,v)).start()
return 0
except Exception as e:
print('Exception! File:[{0}], Line:[{1}]. Exception:{2}'.format(__file__, sys._getframe().f_lineno, e))
return 2
def stop_user_log(self):
'''
stop log.
return:
0: OK
1: exception that driver hasn't started logging files yet.
2: other exception.
'''
rev = 0
try:
if len(self.log_file_rows) == 0:
return 1 # driver hasn't started logging files yet.
for i, (k, v) in enumerate(self.log_files_obj.items()):
v.close()
self.log_file_rows.clear()
self.log_file_names.clear()
self.log_files_obj.clear()
rev = 0
except Exception as e:
print(e)
rev = 2
if self.ws:
time.sleep(1)
self.exit_lock.acquire()
self.exit_thread = True
self.exit_lock.release()
self.ws = False
return rev
# def upload_callback(self, current, total):
# print('({}, {})'.format(current, total))
def upload_azure(self, packet_type, log_file_name):
if self.db_user_access_token == '' or self.sas_token == '':
print("Error: Can not upload log to azure since token is empty! Please check the network.")
print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:'), log_file_name, ' start.')
accountName = 'navview'
countainerName = 'data-1000'
fileName = log_file_name
bcreate_blob_ok = False
error_connection = 'ConnectionError'
error_authorization = 'AuthenticationFailed'
ii=0
while True:
# get data from data_dict.
self.data_lock.acquire()
text = self.data_dict[log_file_name]
self.data_dict[log_file_name] = ''
self.data_lock.release()
# check if user stop logging data.
self.exit_lock.acquire()
if self.exit_thread:
# check for internet and text
if text == '' or (not self.internet_on()):
self.exit_lock.release()
break
else:
pass
self.exit_lock.release()
#let CPU have a break.
if text == '' :
time.sleep(1)
continue
#create blob on azure
if not bcreate_blob_ok:
try:
self.append_blob_service = AppendBlobService(account_name=accountName,
sas_token=self.sas_token,
protocol='http')
self.append_blob_service.create_blob(container_name=countainerName, blob_name=fileName,
content_settings=ContentSettings(content_type='text/plain'))
bcreate_blob_ok = True
threading.Thread(target=self.save_to_db_task, args=(packet_type, log_file_name)).start()
except Exception as e:
# print('Exception when create_blob:', type(e), e)
if error_connection in str(e):
pass
elif error_authorization in str(e):
self.get_sas_token()
self.append_blob_service = AppendBlobService(account_name=accountName,
sas_token=self.sas_token,
protocol='http')
print('Retry to create_blob again...')
continue
# append blob on azure
try:
# self.append_blob_service.append_blob_from_text(countainerName, fileName, text, progress_callback=self.upload_callback)
self.append_blob_service.append_blob_from_text(countainerName, fileName, text)
except Exception as e:
# print('Exception when append_blob:', type(e), e)
if error_connection in str(e):
pass
elif error_authorization in str(e):
self.get_sas_token()
self.append_blob_service = AppendBlobService(account_name=accountName,
sas_token=self.sas_token,
protocol='http')
# if append blob failed, do not drop 'text', but push 'text' to data_dict and re-append next time.
self.data_lock.acquire()
self.data_dict[log_file_name] = text + self.data_dict[log_file_name]
self.data_lock.release()
if bcreate_blob_ok:
# if not self.save_to_ans_platform(packet_type, log_file_name):
# print('save_to_ans_platform failed.')
print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:') , log_file_name, ' done.')
def save_to_db_task(self, packet_type, file_name):
if not self.save_to_ans_platform(packet_type, file_name):
print('save_to_ans_platform failed.')
def update(self, packet, packet_type, is_var_len_frame):
if len(self.log_file_rows) == 0: #if hasn't started logging.
return
if packet_type in self.msgs_need_to_log:
if is_var_len_frame:
self.log_var_len(packet, packet_type)
else:
self.log(packet, packet_type)
def get_log_file_names(self):
return self.log_file_names.copy()
def log(self, data, packet_type):
''' Parse the data, read in from the unit, and generate a data file using
the json properties file to create a header and specify the precision
of the data in the resulting data file.
'''
output_packet = next((x for x in self.output_packets if x['name'] == packet_type), None)
'''Write row of CSV file based on data received. Uses dictionary keys for column titles
'''
if self.log_file_rows[packet_type] == 0:
# Loop through each item in the data dictionary and create a header from the json
# properties that correspond to the items in the dictionary
labels = ''
# for key in data:
for i, (k, v) in enumerate(data.items()):
'''dataStr = output_packet['payload'][i]['name'] + \
' [' + \
output_packet['payload'][i]['unit'] + \
']'''
dataStr = output_packet['payload'][i]['name']
unitStr = output_packet['payload'][i]['unit']
if unitStr == '':
labels = labels + '{0:s},'.format(dataStr)
else:
labels = labels + '{0:s} ({1:s}),'.format(dataStr, unitStr)
# Remove the comma at the end of the string and append a new-line character
labels = labels[:-1]
header = labels + '\n'
else:
header = ''
self.log_file_rows[packet_type] += 1
# Loop through the items in the data dictionary and append to an output string
# (with precision based on the data type defined in the json properties file)
str = ''
for i, (k, v) in enumerate(data.items()):
outputPcktType = output_packet['payload'][i]['type']
if outputPcktType == 'uint32' or outputPcktType == 'int32' or \
outputPcktType == 'uint16' or outputPcktType == 'int16' or \
outputPcktType == 'uint64' or outputPcktType == 'int64':
# integers and unsigned integers
str += '{0:d},'.format(v)
elif outputPcktType == 'double':
# double
str += '{0:0.8f},'.format(v)# 15.12
elif outputPcktType == 'float':
str += '{0:0.4f},'.format(v) # 12.8
elif outputPcktType == 'uint8':
# byte
str += '{0:d},'.format(v)
elif outputPcktType == 'uchar' or outputPcktType == 'char' or outputPcktType == 'string':
# character
str += '{:},'.format(v)
else:
# unknown
str += '{0:3.5f},'.format(v)
#
str = header + str[:-1] + '\n'
self.log_files_obj[packet_type].write(str)
self.log_files_obj[packet_type].flush()
if self.ws:
self.data_lock.acquire()
self.data_dict[self.log_files[packet_type]] = self.data_dict[self.log_files[packet_type]] + str
self.data_lock.release()
def log_var_len(self, data, packet_type):
''' Parse the data, read in from the unit, and generate a data file using
the json properties file to create a header and specify the precision
of the data in the resulting data file.
'''
output_packet = next((x for x in self.output_packets if x['name'] == packet_type), None)
'''Write row of CSV file based on data received. Uses dictionary keys for column titles
'''
if self.log_file_rows[packet_type] == 0:
# Loop through each item in the data dictionary and create a header from the json
# properties that correspond to the items in the dictionary
labels = ''
for value in output_packet['payload']:
dataStr = value['name']
unitStr = value['unit']
if unitStr == '':
labels = labels + '{0:s},'.format(dataStr)
else:
labels = labels + '{0:s} ({1:s}),'.format(dataStr, unitStr)
# Remove the comma at the end of the string and append a new-line character
labels = labels[:-1]
header = labels + '\n'
else:
header = ''
self.log_file_rows[packet_type] += 1
# Loop through the items in the data dictionary and append to an output string
# (with precision based on the data type defined in the json properties file)
str = ''
const_str = ''
var_str = ''
var_fileld_tpyes = []
var_fileld_num = len(output_packet['payload']) - output_packet['var_num']['field_idx']
const_fileld_num = len(output_packet['payload']) - var_fileld_num
for idx, value in enumerate(output_packet['payload']):
if idx >= const_fileld_num:
var_fileld_tpyes.append(value['type'])
for idx, key in enumerate(data):
if idx == 0: # handle const filelds which are all in the first item of data.
for i, (k, v) in enumerate(key.items()):
outputPcktType = output_packet['payload'][i]['type']
if outputPcktType == 'uint32' or outputPcktType == 'int32' or \
outputPcktType == 'uint16' or outputPcktType == 'int16' or \
outputPcktType == 'uint64' or outputPcktType == 'int64':
# integers and unsigned integers
const_str += '{0:d},'.format(v)
elif outputPcktType == 'double':
# double
const_str += '{0:0.12f},'.format(v) # 15.12
elif outputPcktType == 'float':
const_str += '{0:0.4f},'.format(v) # 12.8
elif outputPcktType == 'uint8':
# byte
const_str += '{0:d},'.format(v)
elif outputPcktType == 'uchar' or outputPcktType == 'char' or outputPcktType == 'string':
# character
const_str += '{:},'.format(v)
else:
# unknown
const_str += '{0:3.5f},'.format(v)
else:
for i, (k, v) in enumerate(key.items()):
outputPcktType = var_fileld_tpyes[i]
if outputPcktType == 'uint32' or outputPcktType == 'int32' or \
outputPcktType == 'uint16' or outputPcktType == 'int16' or \
outputPcktType == 'uint64' or outputPcktType == 'int64':
# integers and unsigned integers
var_str += '{0:d},'.format(v)
elif outputPcktType == 'double':
# double
var_str += '{0:15.12f},'.format(v)# 15.12
elif outputPcktType == 'float':
var_str += '{0:12.4f},'.format(v) # 12.8
elif outputPcktType == 'uint8':
# byte
var_str += '{0:d},'.format(v)
elif outputPcktType == 'uchar' or outputPcktType == 'char':
# character
var_str += '{:},'.format(v)
else:
# unknown
var_str += '{0:3.5f},'.format(v)
str = const_str + var_str
str = header + str[:-1] + '\n'
self.log_files_obj[packet_type].write(str)
self.log_files_obj[packet_type].flush()
if self.ws:
self.data_lock.acquire()
self.data_dict[self.log_files[packet_type]] = self.data_dict[self.log_files[packet_type]] + str
self.data_lock.release()
header = ''
str = ''
var_str = ''
def set_user_id(self, user_id):
self.user_id = user_id
if not isinstance(self.user_id, str):
self.user_id = str(self.user_id)
def set_user_access_token(self, access_token):
self.db_user_access_token = access_token
def upload(self, log_file_names):
t = threading.Thread(target=self.upload_to_azure_task, args=(log_file_names, ))
t.start()
def upload_to_azure_task(self, log_files_dict):
self.get_sas_token()
if self.db_user_access_token != '' and self.sas_token != '':
print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:') , 'Start.')
for i, (k, v) in enumerate(log_files_dict.items()): # k: packet type; v: log file name
print('upload:', v)
self.upload_to_azure(k, v)
# self.db_user_access_token = ''
# self.sas_token = ''
print(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S:') , 'Done.')
def get_sas_token(self):
try:
url = self.host_url + "token/storagesas"
headers = {'Content-type': 'application/json', 'Authorization': self.db_user_access_token}
response = requests.post(url, headers=headers)
rev = response.json()
if 'token' in rev:
self.sas_token = rev['token']
else:
self.sas_token = ''
print('Error: Get sas token failed!')
except Exception as e:
print('Exception when get_sas_token:', e)
def upload_to_azure(self, packet_type, file_name):
''' Upload CSV's to Azure container.
'''
f = open("data/" + file_name, "r")
text = f.read() #.decode("utf-8")
try:
self.azure_storage('navview', self.sas_token, 'data', file_name, text)
except Exception as e:
print('azure_storage exception:', e)
return
# Try again!
# self.azure_storage('navview', self.sas_token, 'data', file_name, text)
pass
''' Trigger Database upload
'''
rev = self.save_to_ans_platform(packet_type, file_name)
if not rev:
print('save_to_ans_platform failed.')
def azure_storage(self, accountName, sasToken, countainerName,fileName,text):
if 0:
self.append_blob_service = AppendBlobService(account_name=accountName,
sas_token=sasToken,
protocol='http')
self.append_blob_service.create_blob(container_name=countainerName, blob_name=fileName,
content_settings=ContentSettings(content_type='text/plain'))
self.append_blob_service.append_blob_from_text(countainerName, fileName, text)
else:
self.block_blob_service = BlockBlobService(account_name=accountName,
sas_token=sasToken,
protocol='http')
self.block_blob_service.create_blob_from_text( container_name= countainerName,
blob_name= fileName,
text=text,
content_settings=ContentSettings(content_type='text/plain'))
def save_to_ans_platform(self, packet_type, file_name):
''' Upload CSV related information to the database.
'''
try:
data = {"type": 'INS', "model": 'INS1000', "fileName": file_name, "url": file_name, "userId": self.user_id,
"logInfo": { "pn": '11', "sn": '', "packetType":packet_type,"insProperties":json.dumps(self.rover_properties)}}
url = self.host_url + "api/recordLogs/post"
data_json = json.dumps(data)
headers = {'Content-type': 'application/json', 'Authorization': self.db_user_access_token}
response = requests.post(url, data=data_json, headers=headers)
return True if 'success' in response.json() else False
except Exception as e:
print('Exception when update db:', e)
def internet_on(self):
try:
url = 'https://navview.blob.core.windows.net/'
if sys.version_info[0] > 2:
import urllib.request
response = urllib.request.urlopen(url, timeout=1)
else:
import urllib2
response = urllib2.urlopen(url, timeout=1)
# print(response.read())
return True
except urllib2.URLError as err:
return False
import csv
import sys
import subprocess
import datetime
import time
from azure.storage.blob import AppendBlobService
# Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer
append_blob_service = AppendBlobService(
account_name='storage_account_name',
account_key='storage_account_key')
# Creates an append blob for this app.
append_blob_service.create_container('distances')
append_blob_service.create_blob('distances', 'log.txt')
append_blob_service.append_blob_from_text('distances', 'log.txt', "Starting: " + datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n")
LatLongDict = {}
# Reads the number of jobs from the command line.
jobCount = int(sys.argv[1])
# Reads the airport data in to a list for easy access.
with open('airports-world.csv') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
LatLongDict[row['LocationID']] = [row['Latitude'], row['Longitude']]
import csv
import sys
import subprocess
import datetime
import time
from azure.storage.blob import AppendBlobService
# Configure account name with the Azure Storage Account Name and the account Key from Storage Explorer
append_blob_service = AppendBlobService(account_name='storage_account_name',
account_key='storage_account_key')
# Creates an append blob for this app.
append_blob_service.create_container('distances')
append_blob_service.create_blob('distances', 'log.txt')
append_blob_service.append_blob_from_text(
'distances', 'log.txt', "Starting: " + datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S') + "\n")
LatLongDict = {}
# Reads the number of jobs from the command line.
jobCount = int(sys.argv[1])
# Reads the airport data in to a list for easy access.
with open('airports-world.csv') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
LatLongDict[row['LocationID']] = [row['Latitude'], row['Longitude']]