def set_adress(self, adress):
if len(adress) == self.adress_len:
DataStorage().start_ecc_address = adress.upper()
else:
DataStorage().start_ecc_address = None
self.ui.pushButton_next.setEnabled(
bool(DataStorage().start_ecc_address))
def load_CIFAR10(validation_split=0.9):
NUM_CLASSES = 10
# Wczytanie bazy zdjęć
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
# Wielkości zbiorów danych
TRAIN_SIZE = int(validation_split * len(x_train))
VALIDATION_SIZE = int(len(x_train) - TRAIN_SIZE)
# podział zbioru treningowego na treningowy i walidacyjny
x_validation = x_train[:VALIDATION_SIZE]
x_train = x_train[VALIDATION_SIZE:]
y_validation = y_train[:VALIDATION_SIZE]
y_train = y_train[VALIDATION_SIZE:]
# Zamiana numeru odpowiedzi na macierz labeli
y_train = np_utils.to_categorical(y_train, NUM_CLASSES)
y_validation = np_utils.to_categorical(y_validation, NUM_CLASSES)
y_test = np_utils.to_categorical(y_test, NUM_CLASSES)
input_data = DataStorage(x_train, x_validation, x_test)
output_labels = DataStorage(y_train, y_validation, y_test)
training_data = TrainingData(input_data, output_labels)
return training_data
def make_aditional_null_list(self, result_list_lenght):
one_file_lenght = DataStorage().file_lenght
banks_number = DataStorage().joint_banks_number if DataStorage().banks_type == BanksType.Joint \
else DataStorage().data_banks_number
symbols_in_bank_string = self.get_number_of_symbols_in_bank_string(
self.bits_number)
symbols_in_input_file_string = self.get_number_of_symbols_in_input_file_string(
)
# Добавляем инфу, если объем выходных файлов больше полученных данных
total_string_number = banks_number * one_file_lenght
zero_list = []
if result_list_lenght < total_string_number:
number_of_needed_null_string = total_string_number - result_list_lenght
temp_file = DataStorage().file_dir + 'temp.txt'
zero_list = [
symbols_in_bank_string * '0'
for _ in range(0, number_of_needed_null_string + 1)
]
with open(temp_file, 'w') as f:
total_zero_number = len(zero_list) * symbols_in_bank_string
for _ in range(0, total_zero_number // 2 + 1):
f.write('00\n')
from ECC import ECC
ECC(temp_file).prepare_and_calc_ECC()
def __init__(self, serverhost, serverport, updaterate, lastmessage,
condition):
"""
Constructor.
@param serverhost address to remote server to send data to
@param serverport port of remote server
@param updaterate amount of time before data is sent to remote server
@param lastmessage last message recived from server
@param condition condition synchronization object
"""
threading.Thread.__init__(self)
self.lastmessage = lastmessage
self.condition = condition
self.remotehost = serverhost
self.remoteport = serverport
self.host = get_ip_address('eth1')
print self.host
#self.host = "130.229.145.50"
self.port = 8822
self.go = True
self.id = get_mac()
self.updaterate = updaterate * 10
self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.ds = DataStorage()
def get_next_after(self, page):
if page == self.Start:
return self.SelectFile
elif page == self.SelectFile:
return self.MemoryType
elif page == self.MemoryType:
return self.WithoutECC if DataStorage(
).memory_type == MemoryType.WithoutEcc else self.EccAdress
elif page == self.WithoutECC:
return self.DestinationFolder
elif page == self.EccAdress:
return self.BanksType
elif page == self.BanksType:
return self.SeparatedBanks if DataStorage(
).banks_type == BanksType.Separated else self.JointBanks
elif page == self.JointBanks:
return self.DestinationFolder
elif page == self.SeparatedBanks:
return self.DestinationFolder
elif page == self.DestinationFolder:
return self.Result
elif page == self.Result:
return self.SelectFile
else:
log.error('Unknown page: {}'.format(page))
return page
def prepare_to_open(self):
self._adress = DataStorage().start_ecc_address
self.adress_len = get_number_of_global_adress_symbols(
DataStorage().file_path)
self.ui.lineEdit_ecc_adress.setToolTip('Enter {} hex numbers'.format(
self.adress_len))
super().prepare_to_open()
def __init__(self):
self.S3_BUCKET = Configuration.S3_BUCKET
self.db_handler = DataStorage()
self.sqs = boto3.client('sqs', region_name=Configuration.AWS_REGION)
self.s3 = boto3.client('s3', region_name=Configuration.AWS_REGION)
self.cloudwatch = boto3.client('cloudwatch',
region_name=Configuration.AWS_REGION)
self.messages = []
self.entries = []
self.tweets = []
self.new_tweet = Tweet()
def write_ecc_files(self):
string_number = 0
ecc_bits = DataStorage().ecc_bits
banks_number = DataStorage().ecc_banks_number
ecc_list = get_ecc(DataStorage().file_with_ecc)
# change ecc if ecc_bits is more that 8 bit (2 symbols)
symbols_in_string = int(len(ecc_list[0]))
if symbols_in_string * SIZE_OF_ONE_HEX_CHARACTER_IN_BITS != ecc_bits:
zero_number = ecc_bits // SIZE_OF_ONE_HEX_CHARACTER_IN_BITS - symbols_in_string
finished_output = [
'0' * zero_number + ecc_item for ecc_item in ecc_list
]
else:
finished_output = ecc_list
list_of_files = []
list_dir = os.listdir(self.splitted_files_directory)
file_number = 0
# Open empty files for the writing
for file in list_dir:
if 'ECC' in file:
list_of_files.append(
open(os.path.join(self.splitted_files_directory, file),
'w',
encoding='UTF-8'))
# Put ecc info into banks
for index, value in enumerate(finished_output):
if not index % banks_number:
if not index:
list_of_files[0].write(
'@{0:<5X}{1}'.format(string_number, value) + '\n')
continue
string_number += 1
file_number = index % banks_number
list_of_files[file_number].write(
'@{0:<5X}{1}'.format(string_number, value) + '\n')
# Fill all banks by zero if need (all files must have the same number of string)
if file_number != banks_number - 1:
for index in range(file_number + 1, banks_number):
zero_number = self.get_number_of_symbols_in_bank_string(
bits_number=ecc_bits)
value = '0' * zero_number
list_of_files[index].write(
'@{0:<5X}{1}'.format(string_number, value) + '\n')
for file in list_of_files:
file.close()
def split(self):
if DataStorage().memory_type == MemoryType.WithoutEcc or DataStorage(
).banks_type != BanksType.Separated:
self.create_empty_hex_files(data_banks=self.banks_number)
else:
self.create_empty_hex_files(data_banks=self.banks_number,
ecc_banks=self.ecc_banks_number)
self.write_split_hex_files(self.banks_number)
if DataStorage().memory_type != MemoryType.WithoutEcc and DataStorage(
).banks_type == BanksType.Separated:
self.write_ecc_files()
log.info('All files were written successfully')
def __init__(self, serverhost, serverport, updaterate, lastmessage, condition):
"""
Constructor.
@param serverhost address to remote server to send data to
@param serverport port of remote server
@param updaterate amount of time before data is sent to remote server
@param lastmessage last message recived from server
@param condition condition synchronization object
"""
threading.Thread.__init__(self)
self.lastmessage = lastmessage
self.condition = condition
self.remotehost = serverhost
self.remoteport = serverport
self.host = get_ip_address('eth1')
print self.host
#self.host = "130.229.145.50"
self.port = 8822
self.go = True
self.id = get_mac()
self.updaterate = updaterate*10
self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.ds = DataStorage()
def make_result_list(self):
file_content = get_file_content(self.input_file)
ecc = None
is_ecc_need = DataStorage().memory_type == MemoryType.WithEcc
if is_ecc_need and DataStorage().banks_type == BanksType.Joint:
ecc = get_ecc(DataStorage().file_with_ecc)
result_list = []
symbols_in_file_string = self.get_number_of_symbols_in_input_file_string(
)
symbols_in_bank_string = self.get_number_of_symbols_in_bank_string(
self.bits_number)
number_of_input_string_in_bank_string = int(symbols_in_bank_string /
symbols_in_file_string)
temp_string = ''
ecc_string = ''
counter = 0
for index, string in enumerate(file_content):
# add ecc to string
if is_ecc_need and DataStorage(
).banks_type == BanksType.Joint and ((index + 1) % 4 == 0):
ecc_index = int(((index + 1) / 4)) - 1
try:
ecc_string = '*' + ecc[ecc_index] + '*'
except:
ecc_string = '*' + '00' + '*'
if counter < number_of_input_string_in_bank_string:
temp_string = string + temp_string
counter += 1
else:
if is_ecc_need and DataStorage(
).banks_type == BanksType.Joint and index % 4 == 0:
if self.ecc_location == EccLocation.Right:
temp_string = temp_string + ecc_string
else:
temp_string = ecc_string + temp_string
result_list.append(temp_string)
temp_string = string
counter = 1
self.finished_output = result_list
def extractor(self):
#list of videos to analize
videos_list = os.listdir(self.videos_dir)
#sorting list of videos
videos_list.sort()
#array in which to store videos name
video_name_list = []
#for each video
for i in range(0, len(videos_list)):
#video name
video_name = videos_list[i].split(".")[0]
#adding video name to list
video_name_list.append(video_name)
#video path
video_path = self.videos_dir + '/' + videos_list[i]
#creation of frames folder, if it doesn't already exist
frames_dir = '../frames/' + video_name
if not os.path.exists(frames_dir):
os.mkdir(frames_dir)
print("Directory ", frames_dir, " Created ")
frames_dir += '/'
#creation of jsons folder, if it doesn't already exist
json_dir = '../json_files/' + video_name
if not os.path.exists(json_dir):
os.mkdir(json_dir)
print("Directory ", json_dir, " Created ")
json_dir += '/'
#function to extract frames from the selected video and to store json files with walking data
# self.extraction(video_path, frames_dir, json_dir)
#initialization of cv2 to extract informations from the selected video (width, height, fps, number of frames)
cap = cv2.VideoCapture(video_path)
#width (in pixel) of selected video
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
#height (in pixel) of selected video
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
#frame per second
fps = cap.get(cv2.CAP_PROP_FPS)
#termination of cv2
cap.release()
cv2.destroyAllWindows()
#function to read walking data from json files
pose_estimation = self.getDataByJson(json_dir, width, height, fps)
#function to create txt files with data of interest extracted from each json file
ds = DataStorage(video_name, pose_estimation)
ds.storage()
return video_name_list
def create_banks(self):
try:
Splitter()
except Exception as e:
print('Exception from create_banks: {}'.format(e))
else:
if DataStorage().file_with_ecc is not None and exists(
DataStorage().file_with_ecc):
from shutil import copyfile
from os import remove
copyfile(
DataStorage().file_with_ecc,
DataStorage().splitted_files_directory +
'\\file_with_ecc.txt')
remove(DataStorage().file_with_ecc)
finally:
self.open_next.emit()
def generate_ecc_and_write_to_file(self):
file_with_ecc_name = DataStorage().file_with_ecc = path.join(
DataStorage().programm_location,
'ECC_from_' + DataStorage().file_name)
ecc_file = create_and_open_file(file_with_ecc_name)
global_adress = DataStorage().start_ecc_address
global_adress_bin = hex_to_bin(global_adress)
local_adress_len = 32 - len(str(global_adress_bin))
with open(self.hex_file, 'r') as f:
local_adress = 0
counter = 0
temp_buffer = ''
for line in f:
if counter < 4:
# Write new line in the start of buffer because it's more big bits
temp_buffer = line.replace('\n', '').replace(
'\r', '') + temp_buffer
counter += 1
else:
local_adress_bin = '{0:0{1}b}'.format(
local_adress, local_adress_len)
string_for_ecc = global_adress_bin + local_adress_bin + hex_to_bin(
temp_buffer)
ecc_hex = self.prepare_and_calc_ECC(string_for_ecc)
data_for_file = 'adress:{local} '\
'data:{data} '\
'ecc:{ecc}'.format(local=bin_to_hex(local_adress_bin, min_output_lenght=len(local_adress_bin)/4),
data=temp_buffer,
ecc=ecc_hex
)
ecc_file.write(data_for_file + '\n')
counter = 1
temp_buffer = line.replace('\n', '').replace('\r', '')
local_adress += 1
from Logger import log
log.info('ECC was created in {}'.format(file_with_ecc_name))
ecc_file.close()
def __init__(self):
"""
初始化
:return:
"""
self.processNum = PRECESSNUM
self.accountQueue = Queue()
self.urlQueue = Queue()
self.dataStorage = DataStorage()
self.__dataStorage = self.dataStorage.dataStorage
self.accountName = ACCOUNT.get('name')
self.urlName = LINKEDINURL.get('name')
self.usersName = LINKEDINUSERS.get('name')
self.redis = Redis(self.__dataStorage)
def __init__(self):
super().__init__()
DataStorage()
self.ui = self.initUi()
self.ui.setWindowTitle('VPCreator')
self.init_pages()
self.pages = (self.Start, self.SelectFile, self.MemoryType,
self.WithoutECC, self.EccAdress, self.BanksType,
self.SeparatedBanks, self.JointBanks,
self.DestinationFolder, self.Result)
self.connect_signals()
self.open_page(self.Start)
self.__current_page = self.Start
def current_bits(self, index):
value = self.ui.comboBox_bits_j.itemData(index)
if value:
DataStorage().joint_data_bits = value
def current_amount(self, value):
if value:
DataStorage().joint_banks_number = value
self.fill_rar()
def set_state(self, state): # check using radioButton_right
DataStorage(
).ecc_location = EccLocation.Right if state else EccLocation.Left
def dds():
from DataStorage import DataStorage
return DataStorage('dds', 'dds')
class getData(threading.Thread):
"""
Gets data from sensors and stores them in DataStorage class
"""
def __init__(self, serverhost, serverport, updaterate, lastmessage, condition):
"""
Constructor.
@param serverhost address to remote server to send data to
@param serverport port of remote server
@param updaterate amount of time before data is sent to remote server
@param lastmessage last message recived from server
@param condition condition synchronization object
"""
threading.Thread.__init__(self)
self.lastmessage = lastmessage
self.condition = condition
self.remotehost = serverhost
self.remoteport = serverport
self.host = get_ip_address('eth1')
print self.host
#self.host = "130.229.145.50"
self.port = 8822
self.go = True
self.id = get_mac()
self.updaterate = updaterate*10
self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.ds = DataStorage()
def run(self):
print "Starting data Thread\n"
self.s.bind((self.host, self.port)) # Bind socket to interface
data, addr = self.s.recvfrom(1024) # buffer size is 1024 bytes
update = time.time()
while self.go:
data, addr = self.s.recvfrom(1024) # buffer size is 1024 bytes
#print data
#Stores the data and notifys the streaimg threads that new data is available
self.condition.acquire()
self.lastmessage.storeData(data)
self.condition.notifyAll()
self.condition.release()
self.ds.push(data)
if self.updaterate < time.time()-update:
#Sends data to remote server
t1 = threading.Thread(target = sendData, args=(self.ds.toString(), self.remotehost, self.remoteport))
t1.start()
print "Data to Server"
print self.ds.toString()
update = time.time()
self.s.close()
def stop(self):
self.go = False
def set_label(self):
if DataStorage().destination_folder:
self.ui.label_result.setText(
'{} \nwas splitted.'
'\nClick button to open folder with files'.format(
DataStorage().file_path))
def open_split_files_folder(self):
path = str(DataStorage().splitted_files_directory).replace('/', '\\')
command = 'explorer /open,"{}"'.format(path)
from subprocess import Popen
Popen(command)
def __init__(self):
self.__data_storage = DataStorage()
self.__load_data()
self.__calculate_group_data()
class InOutput():
def __init__(self):
self.__data_storage = DataStorage()
self.__load_data()
self.__calculate_group_data()
def handle_instructions(self, instructions, id):
patient = self.__get_patient_by_id(id)
if(instructions == "get_personal_info"):
if(type(patient) != type(None)):
return self.__get_personal_info(patient)
else:
return None
elif(instructions == "get_icd_category_growth_trend"):
if(type(patient) != type(None)):
return self.__get_icd_category_growth_trend(patient)
else:
return None
elif(instructions == "get_personal_inpatient_cycle"):
if(type(patient) != type(None)):
return self.__get_personal_inpatient_cycle(patient)
else:
return None
elif(instructions == "get_shortterm_mortality_rate"):
if(type(patient) != type(None)):
return self.__get_shortterm_mortality_rate(patient)
else:
return None
elif(instructions == "same_patient_survive_time_mean"):
if(type(patient) != type(None)):
return self.__get_same_patient_survive_time_mean(patient)
else:
return None
elif(instructions == "same_patient_after_confirm_disease_value_counts"):
if(type(patient) != type(None)):
return self.__get_same_patient_after_confirm_disease_value_counts(patient)
else:
return None
elif(instructions == "get_average_of_all_the_patients_cycles"):
return self.__data_storage.get_group_result('avg_of_all_inpatient_cycle')
else:
return None
def __load_data(self):
#讀json file
with open('../data/patient.json','r') as patient_reader:
patient_data = json.load(patient_reader)
with open('../data/record.json','r') as record_reader:
record_data = json.load(record_reader)
with open('../data/inpatient.json','r') as inpatient_reader:
inpatient_data = json.load(inpatient_reader)
patient_df = pd.DataFrame.from_dict(patient_data , orient='index')
record_df = pd.DataFrame.from_dict(record_data , orient='index')
inpatient_df = pd.DataFrame.from_dict(inpatient_data , orient='index')
self.__build_patient_data(patient_df, record_df, inpatient_df)
def __build_patient_data(self,patient_data, record_data, inpatient_data):
#建立patient資料 呼叫parser builder
for id in patient_data['ID'].values:
patient_parser = PatientParser()
patient_parser.parse_patient(patient_data, record_data, inpatient_data, id)
self.__data_storage.add_patient(patient_parser.get_result())
def __calculate_group_data(self):
self.__data_storage.add_calculated_result('avg_of_all_inpatient_cycle',self.__get_average_of_all_the_patients_cycles())
#以下要用visitor做
def __get_personal_info(self, patient):
out = pd.DataFrame(columns = ['id','gender','birthdate'])
out.loc[0] = [patient.get_id(),patient.get_gender(),patient.get_birth()]
out['birthdate'] = out['birthdate'].astype('datetime64')
out['birthdate'] = out['birthdate'].astype('str')
return out
def __get_icd_category_growth_trend(self, patient):
cardPatientVisitor = CalHistoryICDCardVisitor()
patient.accept_visitor(cardPatientVisitor)
position = cardPatientVisitor.get_result()
return position
def __get_personal_inpatient_cycle(self, patient):
cycle_visitor = CalInpatientCycleVisitor()
patient.accept_visitor(cycle_visitor)
return cycle_visitor.get_result()
def __get_average_of_all_the_patients_cycles(self):
patient_list = self.__data_storage.iter_patient_list()
cycle_mean_visitor = CalMeanAllInpatientCycle()
for patient in patient_list:
patient.accept_visitor(cycle_mean_visitor)
return cycle_mean_visitor.get_result()
def __get_shortterm_mortality_rate(self, patient):
quicklyDeathPatientVisitor = CalShortTermMortalityRateVisitor()
patient.accept_visitor(quicklyDeathPatientVisitor)
death_posible = quicklyDeathPatientVisitor.get_result()
return death_posible
def __get_same_patient_survive_time_mean(self, patient):
calSamePatientDeathMeanTimePatientVisitor = CalSamePatientSurviveTimeMeanVisitor(self.__data_storage)
patient.accept_visitor(calSamePatientDeathMeanTimePatientVisitor)
mean = calSamePatientDeathMeanTimePatientVisitor.get_result()
return mean
def __get_same_patient_after_confirm_disease_value_counts(self, patient):
calSamePatientAfterConfimeICDTimesVisitor = CalSamePatientAfterConfimeICDTimesVisitor(self.__data_storage)
patient.accept_visitor(calSamePatientAfterConfimeICDTimesVisitor)
a_d_set = calSamePatientAfterConfimeICDTimesVisitor.get_result()
return a_d_set
def __get_patient_by_id(self, id):
search_patient = None
for patient in self.__data_storage.iter_patient_list():
if id == patient.get_id():
search_patient = patient
break
return search_patient
def data_current_rar(self, index):
value = self.ui.comboBox_data_rar.itemData(index)
if value:
DataStorage().data_rarefaction = value
def data_current_bits(self, index):
value = self.ui.comboBox_data_bits.itemData(index)
if value:
DataStorage().data_bits = value
def data_current_amount(self, value):
if value:
DataStorage().data_banks_number = value
self.fill_data_rar()
def ecc_current_bits(self, index):
value = self.ui.comboBox_ecc_bits.itemData(index)
if value:
DataStorage().ecc_bits = value
def current_rar(self, index):
value = self.ui.comboBox_rar_j.itemData(index)
if value:
DataStorage().joint_data_rarefaction = value
def set_state(self, state): # check using radioButton ecc_off
DataStorage().memory_type = MemoryType.WithoutEcc if state else MemoryType.WithEcc
def ecc_current_amount(self, value):
if value:
DataStorage().ecc_banks_number = value
self.fill_ecc_rar()