def test_error_if_bad_config(self): with self.assertRaises(Exception): CSVConverter(config_json='Not real valid JSON here...') with self.assertRaises(Exception): CSVConverter(config_file_name=os.path.join(fo, 'non-existant-file-hfhffhjksdfhjksfhjk')) with self.assertRaises(Exception): CSVConverter(config_file_name=os.path.join(fo, 'real-but-invalid.json')) with self.assertRaises(Exception): # noinspection PyTypeChecker CSVConverter(config_dict=['This isn', 't a dict'])
def test_simple_case_twice_with_append(self): converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-header-change.json'), append_mode=True) output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple.csv')) with open(os.path.join(fo, 'valid-simple-new.csv')) as output_file: self.assertEqual(output_file.read(), output.replace('\r\n', '\n')) output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple-two.csv')) with open(os.path.join(fo, 'valid-simple-appended.csv')) as output_file: self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def input_csv_to_df(self, input_file, date_range): csv_converter = CSVConverter(self.parser, self.logger) # "TAO_T0N110W_D_ADCP.ascii" => "T0N110W" column_str = input_file.filename.split("_")[1] columns = self.columns(column_str) csv_converter.input_csv_to_df(input_file, date_range, columns, column_str) self.pandas_tools.concat_df(csv_converter.pandas_tools.df)
def input_csv_to_df(self, input_file, date_range, plot_output_path): csv_converter = CSVConverter(self.parser, self.logger) # "0_0.csv" => "0_0" column_str = input_file.filename.split(".")[0] columns = [ column_str + '_' + column for column in ['GHI', 'DNI', 'DHI'] ] csv_converter.input_csv_to_df(input_file, date_range, columns) df = csv_converter.pandas_tools.df.copy() ParserSolarAnywhere.plot(plot_output_path, input_file.filename, df) self.pandas_tools.concat_df(csv_converter.pandas_tools.df)
def run(self): while True: t0 = time.time() # Reconnect very 10 minutes to prevent timeouts, 10 min is arbitrary. if self.ftp is None or self.ticks % 10 == 0: self.__connect_ftp() # Download every file for f in self.files_to_download: file_name = self.__download_ftp(f) # Incidents is different and has its own converter if f == "incidents.xml.gz": with gzip.open(file_name, 'rb') as compressed: with open(file_name[:-3], 'wb') as decompressed: for line in compressed: decompressed.write(line) IncidentConverter(file_name[:-3]) os.remove(file_name[:-3]) # Decompressed file os.remove(file_name) # Downloaded file # Put the file name in a queue so other threads know its done self.queue.put(str(file_name[:-6] + 'txt')) else: with gzip.open(file_name, 'rb') as compressed: with open(file_name[:-3], 'wb') as decompressed: for line in compressed: decompressed.write(line) CSVConverter(file_name[:-3]) os.remove(file_name[:-3]) # Decompressed file os.remove(file_name) # Downloaded file self.queue.put(str(file_name[:-6]) + 'txt') # Wait for a minute executing_time = 60 - (time.time() - t0) if executing_time > 0: time.sleep(60 - (time.time() - t0)) self.ticks += 1
def __init__(self, master = None): Frame.__init__(self, master) self.__master = master self.__init_window() self.__selected_file: str = '' self.__converter = CSVConverter()
def test_xlsx_simple_case_with_junk_before_heading(self): converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-header-change-with-hints.json')) output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple-with-junk-before-heading.xlsx')) with open(os.path.join(fo, 'valid-simple-new.csv')) as output_file: self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def test_xlsx_simple(self): converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-header-change.json')) output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple.xlsx')) with open(os.path.join(fo, 'valid-simple-new.csv')) as output_file: self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def test_append_two_simple(self): converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-header-change.json')) output = converter.convert(input_file_name=[os.path.join(fo, 'valid-simple.csv'), os.path.join(fo, 'valid-simple-two.csv')]) with open(os.path.join(fo, 'valid-simple-appended.csv')) as correct_file: self.assertEqual(correct_file.read(), output.replace('\r\n', '\n'))
def test_simple_with_funlinks(self): converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-with-funlink.json')) output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple.csv')) with open(os.path.join(fo, 'valid-simple-double-digits.csv')) as correct_file: self.assertEqual(correct_file.read(), output.replace('\r\n', '\n'))
def test_simple_with_lambda(self): converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-with-lambda.json')) output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple.csv')) with open(os.path.join(fo, 'valid-simple-to-lower.csv')) as output_file: self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def test_simple_with_defaults(self): converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-with-defaults.json')) output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple-with-missing.csv')) with open(os.path.join(fo, 'valid-simple-with-defaults.csv')) as output_file: self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def test_simple_autogenerated_from_csv(self): output = CSVConverter(no_config=True).generate_json_headers(input_file_name=os.path.join(fo, 'valid-simple.csv')) with open(os.path.join(fo, 'valid-simple-autogenerated.json')) as output_file: self.assertEqual(json.load(output_file), output)
def test_error_if_no_config(self): with self.assertRaises(Exception): CSVConverter(config_json=None, config_file_name=None, config_dict=None) with self.assertRaises(Exception): CSVConverter()
def input_csv_to_df(self, input_file, date_range): csv_converter = CSVConverter(self.parser, self.logger) # "TAO_T5N140W_D_SST_10min.ascii" => "T5N140W" column = input_file.filename.split("_")[1] csv_converter.input_csv_to_df(input_file, date_range, [column]) self.pandas_tools.concat_df(csv_converter.pandas_tools.df)