def test_error_if_bad_config(self):
with self.assertRaises(Exception):
CSVConverter(config_json='Not real valid JSON here...')
with self.assertRaises(Exception):
CSVConverter(config_file_name=os.path.join(fo, 'non-existant-file-hfhffhjksdfhjksfhjk'))
with self.assertRaises(Exception):
CSVConverter(config_file_name=os.path.join(fo, 'real-but-invalid.json'))
with self.assertRaises(Exception):
# noinspection PyTypeChecker
CSVConverter(config_dict=['This isn', 't a dict'])
def test_simple_case_twice_with_append(self):
converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-header-change.json'), append_mode=True)
output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple.csv'))
with open(os.path.join(fo, 'valid-simple-new.csv')) as output_file:
self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple-two.csv'))
with open(os.path.join(fo, 'valid-simple-appended.csv')) as output_file:
self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def input_csv_to_df(self, input_file, date_range):
csv_converter = CSVConverter(self.parser, self.logger)
# "TAO_T0N110W_D_ADCP.ascii" => "T0N110W"
column_str = input_file.filename.split("_")[1]
columns = self.columns(column_str)
csv_converter.input_csv_to_df(input_file, date_range, columns,
column_str)
self.pandas_tools.concat_df(csv_converter.pandas_tools.df)
def input_csv_to_df(self, input_file, date_range, plot_output_path):
csv_converter = CSVConverter(self.parser, self.logger)
# "0_0.csv" => "0_0"
column_str = input_file.filename.split(".")[0]
columns = [
column_str + '_' + column for column in ['GHI', 'DNI', 'DHI']
]
csv_converter.input_csv_to_df(input_file, date_range, columns)
df = csv_converter.pandas_tools.df.copy()
ParserSolarAnywhere.plot(plot_output_path, input_file.filename, df)
self.pandas_tools.concat_df(csv_converter.pandas_tools.df)
def run(self):
while True:
t0 = time.time()
# Reconnect very 10 minutes to prevent timeouts, 10 min is arbitrary.
if self.ftp is None or self.ticks % 10 == 0:
self.__connect_ftp()
# Download every file
for f in self.files_to_download:
file_name = self.__download_ftp(f)
# Incidents is different and has its own converter
if f == "incidents.xml.gz":
with gzip.open(file_name, 'rb') as compressed:
with open(file_name[:-3], 'wb') as decompressed:
for line in compressed:
decompressed.write(line)
IncidentConverter(file_name[:-3])
os.remove(file_name[:-3]) # Decompressed file
os.remove(file_name) # Downloaded file
# Put the file name in a queue so other threads know its done
self.queue.put(str(file_name[:-6] + 'txt'))
else:
with gzip.open(file_name, 'rb') as compressed:
with open(file_name[:-3], 'wb') as decompressed:
for line in compressed:
decompressed.write(line)
CSVConverter(file_name[:-3])
os.remove(file_name[:-3]) # Decompressed file
os.remove(file_name) # Downloaded file
self.queue.put(str(file_name[:-6]) + 'txt')
# Wait for a minute
executing_time = 60 - (time.time() - t0)
if executing_time > 0:
time.sleep(60 - (time.time() - t0))
self.ticks += 1
def __init__(self, master = None): Frame.__init__(self, master) self.__master = master self.__init_window() self.__selected_file: str = '' self.__converter = CSVConverter()
def test_xlsx_simple_case_with_junk_before_heading(self):
converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-header-change-with-hints.json'))
output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple-with-junk-before-heading.xlsx'))
with open(os.path.join(fo, 'valid-simple-new.csv')) as output_file:
self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def test_xlsx_simple(self):
converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-header-change.json'))
output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple.xlsx'))
with open(os.path.join(fo, 'valid-simple-new.csv')) as output_file:
self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def test_append_two_simple(self):
converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-header-change.json'))
output = converter.convert(input_file_name=[os.path.join(fo, 'valid-simple.csv'),
os.path.join(fo, 'valid-simple-two.csv')])
with open(os.path.join(fo, 'valid-simple-appended.csv')) as correct_file:
self.assertEqual(correct_file.read(), output.replace('\r\n', '\n'))
def test_simple_with_funlinks(self):
converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-with-funlink.json'))
output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple.csv'))
with open(os.path.join(fo, 'valid-simple-double-digits.csv')) as correct_file:
self.assertEqual(correct_file.read(), output.replace('\r\n', '\n'))
def test_simple_with_lambda(self):
converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-with-lambda.json'))
output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple.csv'))
with open(os.path.join(fo, 'valid-simple-to-lower.csv')) as output_file:
self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def test_simple_with_defaults(self):
converter = CSVConverter(config_file_name=os.path.join(fo, 'valid-simple-with-defaults.json'))
output = converter.convert(input_file_name=os.path.join(fo, 'valid-simple-with-missing.csv'))
with open(os.path.join(fo, 'valid-simple-with-defaults.csv')) as output_file:
self.assertEqual(output_file.read(), output.replace('\r\n', '\n'))
def test_simple_autogenerated_from_csv(self):
output = CSVConverter(no_config=True).generate_json_headers(input_file_name=os.path.join(fo, 'valid-simple.csv'))
with open(os.path.join(fo, 'valid-simple-autogenerated.json')) as output_file:
self.assertEqual(json.load(output_file), output)
def test_error_if_no_config(self):
with self.assertRaises(Exception):
CSVConverter(config_json=None, config_file_name=None, config_dict=None)
with self.assertRaises(Exception):
CSVConverter()
def input_csv_to_df(self, input_file, date_range):
csv_converter = CSVConverter(self.parser, self.logger)
# "TAO_T5N140W_D_SST_10min.ascii" => "T5N140W"
column = input_file.filename.split("_")[1]
csv_converter.input_csv_to_df(input_file, date_range, [column])
self.pandas_tools.concat_df(csv_converter.pandas_tools.df)