def statistic_iterate_all(db, liste, batchsize=50, visual=False): """ Iterates over all subcomponents in the list and evaluates the above tests, visualizes the tests and returns dataframe with percentage of stationary signals for every subcomponent in the dataset and dict with filename as key and dataframe as values.(1 if stationary) :param db: Database you want to iterate :type db: Database :param liste: list of all subcomponents :type liste: list :return: df with percentage of stationary signals for every subcomponent in the dataset, dict with filename as key """ files = [] for i, (name, dct) in enumerate(iterDict(db, liste, querytype='or')): files.append([name, dct]) print(i) if i % batchsize == 0: p = Pool(processes=1) worker = partial(statistic_iterate_all_worker, liste) p.map(worker, files) p.close() files = [] p = Pool(processes=1) worker = partial(statistic_iterate_all_worker, liste) p.map(worker, files) p.close() return
def granger_df_producer(db, liste, stationary_path=cd.paths.stationary_results, causality_path=cd.paths.causality_results, batchsize=100): """ Produces a Dataframe with the results of the granger test iterated over all files :param db: Database you want to iterate :type db: Database :param liste: list of subcomponents :type liste: list :return: df with results of granger test """ stationary_path = os.path.realpath(stationary_path) causality_path = os.path.realpath(causality_path) combinations = make_combinations(liste) worker_input = [] for i, (file_name, dct) in enumerate(iterDict(db, liste, querytype='or', linenr=True)): try: file_path = os.path.join(stationary_path, file_name) save_path = os.path.join(causality_path, file_name) worker_input.append([ file_name, file_path, save_path, stationary_path, dct, combinations, liste, causality_path ]) except: print('couldnt append file to chucklist') continue print(i) if i % batchsize == 0 and i >= 1: p = Pool(processes=12) p.map(granger_df_producer_worker, worker_input) p.close() worker_input = [] p = Pool(processes=12) p.map(granger_df_producer_worker, worker_input) p.close()
#most_subcompos = pickle.load(open('most_subcomponents.p', 'rb')).index[:99] #most_subcompos = list(most_subcompos) start = time.time() n = 0 cd.data.store(n, 'test') combinations = [] for X in most_subcompos: for Y in most_subcompos: if (X, Y) in combinations or (Y, X) in combinations or X == Y: continue combinations.append((X, Y)) output = [] for file_name, dct in iterDict(db, most_subcompos, querytype='or', linenr=True): n = n + 1 print(n) if file_name not in os.listdir(stationary_path): continue else: stationary_file = pickle.load( open(os.path.join(stationary_path, file_name), 'rb')) for X, Y in combinations: if X in dct and Y in dct: if stationary_file[X][0] == 1 and stationary_file[X][1] == 1: ergebnisX = True else: ergebnisX = False
#most_subcompos = ['ATT_DesRoll', 'ATT_Roll'] most_subcompos = pickle.load(open('most_subcomponents.p', 'rb')).index[:99] most_subcompos = list(most_subcompos) n = 0 combinations = [] for X in most_subcompos: for Y in most_subcompos: if (X, Y) in combinations or (Y, X) in combinations or X == Y: continue combinations.append((X, Y)) output = [] start = time.time() for file_name, dct in iterDict(db, most_subcompos, querytype='or'): n = n + 1 #print(n) for X, Y in combinations: if X in dct and Y in dct: dataX = pd.DataFrame(data=dct[X], index=dct[X.split('_')[0] + '_line_number'], columns=[X]) dataY = pd.DataFrame(data=dct[Y], index=dct[Y.split('_')[0] + '_line_number'], columns=[Y]) data = pd.concat([dataX, dataY], axis=1) data = Database.fillna(data) try: output.append(Granger.test(data, 0.05, True, True)) except:
from CrashDiag.Preproc.Database import Database, iterDict from CrashDiag.Preproc.SegDatabase import SegDatabase from CrashDiag.Preproc.Segmentation import plot_segments import matplotlib.pyplot as plt import numpy as np import pandas as pd db = Database() subcomponents = ['ATT_Yaw', 'ATT_DesYaw'] for file_name, file in iterDict(db, subcomponents, querytype='and', timestamp=True): ATT_YawDiff = np.column_stack((file['ATT_Yaw'], -file['ATT_DesYaw'])).sum(axis=1) ATT_YawDiff = pd.DataFrame(data=ATT_YawDiff, index=file['ATT_Yaw'].index) ATT_YawDiff.columns = ['ATT_YawDiff'] ATT_YawDiff = ATT_YawDiff % 360 ATT_YawDiff[ATT_YawDiff>180] = ATT_YawDiff[ATT_YawDiff>180] - 360 plt.plot(ATT_YawDiff) plt.show() pass #seg = SegDatabase.segment_and_pickle_one_log('', file_name, file, ['ATT_YawDiff'], return_obj=True) #plot_segments(seg['ATT_YawDiff']) #plt.show()