def open_stream(self, file_path, feed_format, mode='r'):
if self.errors.error_occured:
return None
encoding = feed_format['encoding']
self.file_handler = Files(self.errors)
self.file_handler.open_file(file_path, mode, encoding)
def table2csv(self, table, columns, file_path, file_format):
if self._errors.error_occured:
return None
list_separator = file_format['list_separator']
decimal_symbol = file_format['decimal_symbol']
encoding = file_format['encoding']
file_column_formats = file_format['file_column_formats']
tools = Tools(self._errors)
file = Files(self._errors)
column_formats = tools.shape_column_formats(columns,
file_column_formats)
file.open_file(file_path, 'w', encoding)
line = tools.implode(columns, list_separator)
file.write_line(line)
length = len(table[columns[0]])
for rec_cnt in range(length):
rec = tools.get_rec_from_table(rec_cnt, table)
tools.str_rec(rec, column_formats)
line = tools.rec2line(rec, columns, list_separator)
file.write_line(line)
file.close_file()
def csv2table(self, file_path, file_format):
if self._errors.error_occured:
return None
list_separator = file_format['list_separator']
decimal_symbol = file_format['decimal_symbol']
encoding = file_format['encoding']
column_types = file_format['column_types']
tools = Tools(self._errors)
file = Files(self._errors)
table = {}
columns = []
file.open_file(file_path, 'r', encoding)
while not self._errors.error_occured:
line = file.read_line()
if line:
line = line.strip('\n')
columns = tools.explode(line, list_separator)
else:
self._errors.raise_error('File ' + file_path + ' is empty')
break
column_types = tools.shape_column_types(columns, column_types)
for col in columns:
table[col] = []
while not self._errors.error_occured:
line = file.read_line()
if line:
line = line.strip('\n')
if line:
rec = tools.line2rec(line, columns, list_separator)
tools.type_rec(rec, column_types)
tools.add_rec_to_table(rec, table)
else:
break
file.close_file()
return table, columns
def split_text(self, file_path, enc, separators):
if self.errors.error_occured:
return None
for cnt in range(len(separators)):
separators[cnt] = self.name2symbol(separators[cnt])
input_file = Files(self.errors)
input_file.open_file(file_path, 'r' ,enc)
sep = ''
wrd = ''
self.text_elements = []
last_smb_type = 's'
smb_cnt = 0
while not self.errors.error_occured:
smb = input_file.read_smb(1)
if smb == '':
break
else:
if smb in separators:
if last_smb_type == 'l':
self.text_elements.append(wrd)
wrd = ''
sep += smb
last_smb_type = 's'
else:
if last_smb_type == 's':
self.text_elements.append(sep)
sep = ''
wrd += smb
last_smb_type = 'l'
smb_cnt += 1
if sep != '':
self.text_elements.append(sep)
if wrd != '':
self.text_elements.append(wrd)
self.text_elements.append('')
input_file.close_file()
def read_ini(self, ini_file_path, encoding):
if self.errors.error_occured:
return None
ini_file = Files(self.errors)
ini_file.open_file(ini_file_path, 'r', encoding)
last_section = None
while not self.errors.error_occured:
line = ini_file.read_line()
if line:
line = self.exlude_non_data(line)
section, param, value = self.parse_line(line)
if section:
self.settings[section] = {}
last_section = section
if last_section and param:
self.settings[last_section][param] = value
else:
break
ini_file.close_file()
self.settings['ini_file_path'] = ini_file_path
return self.settings
class DataStream:
def __init__(self, errors):
self.errors = errors
self.file_handler = None
self.tools = Tools(self.errors)
def open_stream(self, file_path, feed_format, mode='r'):
if self.errors.error_occured:
return None
encoding = feed_format['encoding']
self.file_handler = Files(self.errors)
self.file_handler.open_file(file_path, mode, encoding)
def type_value(self, value, type):
if value:
if type == 'num':
return float(value)
elif type == 'int':
return int(value)
elif type == 'float':
return float(value)
elif type == 'yyyymmdd':
return {
'yyyymmdd': value,
'yyyy': value[0:4],
'yy': value[2:4],
'mm': value[4:6],
'dd': value[6:]
}
elif type == 'hhmmss':
return {
'hhmmss': value,
'hh': value[0:2],
'mm': value[2:4],
'ss': value[4:]
}
else:
value = None
return value
def type_record(self, rec, columns, types):
for col_cnt in range(len(rec)):
rec[columns[col_cnt]] = self.type_value(rec[columns[col_cnt]],
types[col_cnt])
def read_all(self, feed_format):
if self.errors.error_occured:
return {}
skip_first_lines_number = feed_format['header_lines_number']
columns = feed_format['columns']
column_separator = feed_format['column_separator']
column_data_types = feed_format['column_data_types']
while skip_first_lines_number > 0 and not self.errors.error_occured:
line = self.file_handler.read_line()
skip_first_lines_number -= 1
data = {}
for col in columns:
data[col] = []
while not self.errors.error_occured:
line = self.file_handler.read_line()
if line:
line = line.rstrip('\n')
if line:
rec = self.tools.line_to_record(column_separator, line,
columns)
self.type_record(rec, columns, column_data_types)
for col_cnt in range(len(rec)):
data[columns[col_cnt]].append(rec[columns[col_cnt]])
else:
break
return data
def to_str(self, value, type):
if value != None:
if type == 'num':
return str(value)
elif type == 'int':
return str(value)
elif type == 'float':
return str(value)
elif type == 'date':
return value['yyyy'] + value['mm'] + value['dd']
elif type == 'time':
return value['hh'] + value['mm'] + value['ss']
elif type == 'yyyymmdd':
return value
elif type == 'hhmmss':
return value
else:
value = ''
return value
def write_all(self, data, feed_format):
if self.errors.error_occured:
return None
columns = feed_format['columns']
column_separator = feed_format['column_separator']
column_data_types = feed_format['column_data_types']
header_lines_number = feed_format['header_lines_number']
if header_lines_number > 0:
line = ''
for col in columns:
line += column_separator + col
line = line[len(column_separator):]
self.file_handler.write_line(line)
if len(columns) > 0:
for cnt in range(len(data[columns[0]])):
line = ''
col_cnt = 0
for col in columns:
line += column_separator + self.to_str(
data[col][cnt], column_data_types[col_cnt])
col_cnt += 1
line = line[len(column_separator):]
self.file_handler.write_line(line)
def close_stream(self):
if self.file_handler and not self.errors.error_occured:
self.file_handler.close_file()