def goldenSet(number): # load the filtered dataset filData = qbPre.readSimpleFile('data/write/fil_comb_results.csv'); ranSample = random.sample(filData, number); qbPre.writeFilCSV('data/relAnalytics/goldenSet.csv',ranSample);
def filterData(): filData = []; # to store the filtered dataset count = 0; # to keep the counter # read original file and load the relevant data # list all the csv log files with records paths = qbPre.listFiles(qbGbl.oriFileName); # foreach csv file in the paths for path in paths: filterSet = qbPre.filterFile(path,count); # filter data filData.extend(filterSet[0]); # add to the filtered dataset in RAM count = filterSet[1]; # update count # write filtered data to a different file in the HDD qbPre.writeFilCSV(qbGbl.filFileName,filData);
def filterData(): filData = [] # to store the filtered dataset count = 0 # to keep the counter # read original file and load the relevant data # list all the csv log files with records paths = qbPre.listFiles(qbGbl.oriFileName) # foreach csv file in the paths for path in paths: filterSet = qbPre.filterFile(path, count) # filter data filData.extend(filterSet[0]) # add to the filtered dataset in RAM count = filterSet[1] # update count # write filtered data to a different file in the HDD qbPre.writeFilCSV(qbGbl.filFileName, filData)