def filter_dataset(original_path,
original_filename,
csv_writer,
algorithms_array,
print_header=False):
csv_reader = CSVReader(original_path, original_filename)
matching_algorithm = False
while csv_reader.continue_reading:
line = csv_reader.read_line()
filename, algorithm_name = line[1], line[3]
if len(algorithm_name) > 0:
if matching_algorithm and algorithm_name not in algorithms_array:
matching_algorithm = False
elif not matching_algorithm and algorithm_name in algorithms_array:
matching_algorithm = True
if line[0] == "Dataset":
if print_header:
csv_writer.write_row(line)
elif len(filename) > 0 or matching_algorithm:
csv_writer.write_row(line)
csv_reader.close()
def compare_pca_results():
original = CSVReader(pca_mask_folder, "results-original.csv")
results = CSVReader(pca_mask_folder, "results-mask.csv")
csv_writer = CSVWriter(pca_mask_folder, "results-merge.csv")
assert (original.total_lines == results.total_lines)
while original.continue_reading:
original_line = original.read_line()
results_line = results.read_line()
coder_name = original_line[3]
if coder_name in ["Coder", "CoderBase"]:
# assert(original_line == results_line)
csv_writer.write_row(parse_line(original_line))
else:
if len(original_line[4]) > 0: # algorithm params line
original_line, results_line = write_params_line(
csv_writer, original_line, results_line)
original_line = parse_line(original_line)
results_line = parse_mask_line(results_line)
# if original_line == results_line:
# continue
csv_writer.write_row(['O'] + original_line[1:] + ['O'])
csv_writer.write_row(['M'] + results_line[1:] + ['M'])
diff = []
for i in range(7, len(original_line)):
original_value, results_value = original_line[i], results_line[
i]
if i % 2 == 0:
# floats (percentages)
# diff_value = results_value + ' - ' + original_value
diff_value = float(results_value) - float(original_value)
diff_value = diff_scenarios(diff_value)
else:
# integers (bits)
if original_value == results_value:
diff_value = '==='
else:
diff_value = c_int(results_value) - c_int(
original_value)
diff_value = '+++' + str(
diff_value) if diff_value > 0 else '---' + str(
diff_value)
diff.append(diff_value)
diff_line = original_line[:7] + diff
csv_writer.write_row(diff_line)
original.close()
results.close()
csv_writer.close()
def read_csv(expected_rows):
csv_reader = CSVReader(path, filename)
it = 0
while csv_reader.continue_reading:
row = csv_reader.read_row()
expected_row = expected_rows[it]
if expected_row != row:
print 'row', row, 'expected_row', expected_row
raise AssertionError('ERROR.')
it += 1
csv_reader.close()
def clean():
input_path = OSUtils.cpp_project_path() + "/test_files"
input_filename = "noaa-adcp-201501.csv"
output_path = OSUtils.cpp_project_path() + "/test_files"
output_filename = "noaa-adcp-201501_2cols.csv"
csv_reader = CSVReader(input_path, input_filename)
csv_writer = CSVWriter(output_path, output_filename)
iterate(csv_reader, csv_writer)
csv_reader.close()
csv_writer.close()
def compare():
test_files_path = OSUtils.cpp_project_path() + "/test_files"
csv_reader_1 = CSVReader(OSUtils.datasets_csv_path() + "[1]irkis/",
"vwc_1202.dat.csv")
csv_reader_2 = CSVReader(test_files_path,
"vwc_1202.dat.csv-CoderSF-Decode.csv")
while csv_reader_1.continue_reading:
assert (
csv_reader_1.current_line_count == csv_reader_2.current_line_count)
current_line_count = csv_reader_1.current_line_count
line_1 = csv_reader_1.read_line()
line_2 = csv_reader_2.read_line()
compare_lines(line_1, line_2, current_line_count)
print "COMPARE SUCCESS!!"
csv_reader_1.close()
csv_reader_2.close()
def csv_row_count(cls, input_path, input_filename):
csv = CSVReader(input_path, input_filename)
csv.close()
header_rows = CSVReader.first_data_row(input_path, input_filename) + 1
return csv.total_lines - header_rows
import sys
sys.path.append('.')
from file_utils.csv_utils.csv_writer import CSVWriter
from file_utils.csv_utils.csv_reader import CSVReader
path = "/Users/pablocerve/Documents/FING/Proyecto/pc-tesis/dataset_parser/scripts/informe/results/3.1/06.2020/2-complete"
file = "results-mm3.csv"
new_file = "results-mm3-p.csv"
reader = CSVReader(path, file)
writer = CSVWriter(path, new_file)
while reader.continue_reading:
line = reader.read_line()
if "0" in line[4]:
line[4] = int(float(line[4]) * 100)
writer.write_row(line)
writer.close()
reader.close()
def calculate_stds(cls, input_path, input_filename, first_data_row):
csv_reader = CSVReader(input_path, input_filename)
counts, means = cls._calculate_means(csv_reader, first_data_row)
stds = cls._calculate_stds(csv_reader, counts, means)
csv_reader.close()
return stds