def set_params(self, args):
tools = Tools(self.errors)
self.read_settings(args)
self.settings['g1'] = self.ini_parser.get_param('g1', None)
self.settings['g2'] = {}
self.settings['g2']['p1'] = tools.explode(
',', self.ini_parser.get_param('g2', 'p1'))
self.settings['g3'] = {}
self.settings['g3']['i4'] = self.ini_parser.get_param(
'g3', 'i4', 'int')
def table2csv(self, table, columns, file_path, file_format):
if self._errors.error_occured:
return None
list_separator = file_format['list_separator']
decimal_symbol = file_format['decimal_symbol']
encoding = file_format['encoding']
file_column_formats = file_format['file_column_formats']
tools = Tools(self._errors)
file = Files(self._errors)
column_formats = tools.shape_column_formats(columns,
file_column_formats)
file.open_file(file_path, 'w', encoding)
line = tools.implode(columns, list_separator)
file.write_line(line)
length = len(table[columns[0]])
for rec_cnt in range(length):
rec = tools.get_rec_from_table(rec_cnt, table)
tools.str_rec(rec, column_formats)
line = tools.rec2line(rec, columns, list_separator)
file.write_line(line)
file.close_file()
def is_non_working_day(self, current_trading_day):
ctd_year = str(current_trading_day.year)
ctd_month = str(current_trading_day.month)
ctd_day = str(current_trading_day.day)
tools = Tools(self.errors)
nw_days = tools.explode(
',',
self.tc_ini_parser.get_param('non_working_days',
ctd_month + '.' + ctd_year))
if ctd_day in nw_days:
return True
return False
def csv2table(self, file_path, file_format):
if self._errors.error_occured:
return None
list_separator = file_format['list_separator']
decimal_symbol = file_format['decimal_symbol']
encoding = file_format['encoding']
column_types = file_format['column_types']
tools = Tools(self._errors)
file = Files(self._errors)
table = {}
columns = []
file.open_file(file_path, 'r', encoding)
while not self._errors.error_occured:
line = file.read_line()
if line:
line = line.strip('\n')
columns = tools.explode(line, list_separator)
else:
self._errors.raise_error('File ' + file_path + ' is empty')
break
column_types = tools.shape_column_types(columns, column_types)
for col in columns:
table[col] = []
while not self._errors.error_occured:
line = file.read_line()
if line:
line = line.strip('\n')
if line:
rec = tools.line2rec(line, columns, list_separator)
tools.type_rec(rec, column_types)
tools.add_rec_to_table(rec, table)
else:
break
file.close_file()
return table, columns
def set_params(self, args):
tools = Tools(self.errors)
self.read_settings(args)
self.settings['common'] = {}
self.settings['common']['time_frames'] = tools.explode(
',', self.ini_parser.get_param('common', 'time_frames'))
self.settings['common']['output_folder'] = self.ini_parser.get_param(
'common', 'output_folder')
self.settings['common'][
'trading_calendar'] = self.ini_parser.get_param(
'common', 'trading_calendar')
self.settings['common'][
'always_update_past_working_days_number'] = self.ini_parser.get_param(
'common', 'always_update_past_working_days_number', 'int')
self.tc_ini_parser.read_ini(
self.settings['common']['trading_calendar'], self.ini_encoding)
# self.settings['non_working_days'] = {}
# self.settings['non_working_days'] = self.tc_ini_parser.get_param('non_working_days')
self.settings['contracts'] = {}
tickers = tools.explode(
',', self.ini_parser.get_param('contracts', 'tickers'))
for ticker_idx in range(len(tickers)):
self.settings['contracts'][ticker_idx] = {}
self.settings['contracts'][ticker_idx]['ticker'] = tickers[
ticker_idx]
self.settings['contracts'][ticker_idx]['list'] = {}
list = tools.explode(
',', self.ini_parser.get_param('contracts',
tickers[ticker_idx]))
for contract_idx in range(len(list)):
self.settings['contracts'][ticker_idx]['list'][
contract_idx] = self.ini_parser.get_param(
list[contract_idx])
def __init__(self, errors):
self.errors = errors
self.file_handler = None
self.tools = Tools(self.errors)
class DataStream:
def __init__(self, errors):
self.errors = errors
self.file_handler = None
self.tools = Tools(self.errors)
def open_stream(self, file_path, feed_format, mode='r'):
if self.errors.error_occured:
return None
encoding = feed_format['encoding']
self.file_handler = Files(self.errors)
self.file_handler.open_file(file_path, mode, encoding)
def type_value(self, value, type):
if value:
if type == 'num':
return float(value)
elif type == 'int':
return int(value)
elif type == 'float':
return float(value)
elif type == 'yyyymmdd':
return {
'yyyymmdd': value,
'yyyy': value[0:4],
'yy': value[2:4],
'mm': value[4:6],
'dd': value[6:]
}
elif type == 'hhmmss':
return {
'hhmmss': value,
'hh': value[0:2],
'mm': value[2:4],
'ss': value[4:]
}
else:
value = None
return value
def type_record(self, rec, columns, types):
for col_cnt in range(len(rec)):
rec[columns[col_cnt]] = self.type_value(rec[columns[col_cnt]],
types[col_cnt])
def read_all(self, feed_format):
if self.errors.error_occured:
return {}
skip_first_lines_number = feed_format['header_lines_number']
columns = feed_format['columns']
column_separator = feed_format['column_separator']
column_data_types = feed_format['column_data_types']
while skip_first_lines_number > 0 and not self.errors.error_occured:
line = self.file_handler.read_line()
skip_first_lines_number -= 1
data = {}
for col in columns:
data[col] = []
while not self.errors.error_occured:
line = self.file_handler.read_line()
if line:
line = line.rstrip('\n')
if line:
rec = self.tools.line_to_record(column_separator, line,
columns)
self.type_record(rec, columns, column_data_types)
for col_cnt in range(len(rec)):
data[columns[col_cnt]].append(rec[columns[col_cnt]])
else:
break
return data
def to_str(self, value, type):
if value != None:
if type == 'num':
return str(value)
elif type == 'int':
return str(value)
elif type == 'float':
return str(value)
elif type == 'date':
return value['yyyy'] + value['mm'] + value['dd']
elif type == 'time':
return value['hh'] + value['mm'] + value['ss']
elif type == 'yyyymmdd':
return value
elif type == 'hhmmss':
return value
else:
value = ''
return value
def write_all(self, data, feed_format):
if self.errors.error_occured:
return None
columns = feed_format['columns']
column_separator = feed_format['column_separator']
column_data_types = feed_format['column_data_types']
header_lines_number = feed_format['header_lines_number']
if header_lines_number > 0:
line = ''
for col in columns:
line += column_separator + col
line = line[len(column_separator):]
self.file_handler.write_line(line)
if len(columns) > 0:
for cnt in range(len(data[columns[0]])):
line = ''
col_cnt = 0
for col in columns:
line += column_separator + self.to_str(
data[col][cnt], column_data_types[col_cnt])
col_cnt += 1
line = line[len(column_separator):]
self.file_handler.write_line(line)
def close_stream(self):
if self.file_handler and not self.errors.error_occured:
self.file_handler.close_file()
def __init__(self, errors):
self.errors = errors
self.settings = {}
self.tools = Tools(self.errors)
class IniParser:
def __init__(self, errors):
self.errors = errors
self.settings = {}
self.tools = Tools(self.errors)
def exlude_non_data(self, line):
line = line.rstrip('\n')
line = line.rstrip('\r')
start_comment_pos = line.find(';')
if start_comment_pos > -1:
line = line[0:start_comment_pos]
line = line.replace('\t', '')
line = line.replace(' ', '')
return line
def parse_line(self, line):
section = None
param = None
value = None
if line[0:1] == '[' and line[-1:] == ']':
section = line[1:-1]
else:
eq_smb = line.find('=')
param = line[0:eq_smb]
value = line[eq_smb + 1:]
return section, param, value
def read_ini(self, ini_file_path, encoding):
if self.errors.error_occured:
return None
ini_file = Files(self.errors)
ini_file.open_file(ini_file_path, 'r', encoding)
last_section = None
while not self.errors.error_occured:
line = ini_file.read_line()
if line:
line = self.exlude_non_data(line)
section, param, value = self.parse_line(line)
if section:
self.settings[section] = {}
last_section = section
if last_section and param:
self.settings[last_section][param] = value
else:
break
ini_file.close_file()
self.settings['ini_file_path'] = ini_file_path
return self.settings
def get_param(self, section, param=None, param_type='str', sep=',', error_ignoring=False, default_param_value=None):
if self.errors.error_occured:
if param:
return ''
else:
return {}
if self.settings.get(section):
if param:
if self.settings[section].get(param):
return self.tools.str2type(self.settings[section][param], param_type, sep)
else:
if not error_ignoring:
self.errors.raise_error('No parameter [' + section + '][' + param + '] in ini file ' + self.settings['ini_file_path'])
return ''
else:
return self.tools.str2type(default_param_value, param_type, sep)
else:
return self.settings[section]
else:
if not error_ignoring:
self.errors.raise_error('No section [' + section + '] in ini file ' + self.settings['ini_file_path'])
return {}
def set_params(self, args):
self.read_settings(args)
tools = Tools(self.errors)
self.settings['input'] = {}
self.settings['input']['file_path'] = self.ini_parser.get_param('input', 'file_path')
self.settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
self.settings['output'] = {}
self.settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
input_feed_format = self.settings['input']['input_feed_format']
self.settings[input_feed_format] = {}
self.settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
self.settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
self.settings[input_feed_format]['columns'] = tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
self.settings[input_feed_format]['column_separator'] = tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
self.settings[input_feed_format]['column_data_types'] = tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
self.settings['plotter'] = {}
self.settings['plotter']['fig_folder'] = self.settings['output']['folder'] + self.ini_parser.get_param('plotter', 'fig_folder')
self.settings['plotter']['x_ticks_data_columns'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks_data_columns'))
self.settings['plotter']['x_ticks'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks'))
self.settings['plotter']['x_labels_format'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_labels_format'))
self.settings['plotter']['series'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series'))
self.settings['plotter']['subplot_height_share'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'subplot_height_share')))
self.settings['plotter']['subplots_number'] = self.ini_parser.get_param('plotter', 'subplots_number', 'int')
self.settings['plotter']['seria_to_subbplot_binding'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'seria_to_subbplot_binding')))
def clone_value(value, number):
return number * [value]
ignoring_missed_data_line = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'ignoring_missed_data_line'), len(self.settings['plotter']['series'])))
series_linewidth = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_linewidth'), len(self.settings['plotter']['series'])))
series_markersize = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_markersize'), len(self.settings['plotter']['series'])))
series_marker = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_marker'), len(self.settings['plotter']['series'])))
series_color = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_color'), len(self.settings['plotter']['series'])))
series_alpha = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_alpha'), len(self.settings['plotter']['series'])))
self.settings['plotter']['ignoring_missed_data_line'] = tools.bool_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'ignoring_missed_data_line', error_ignoring = True, default_param_value = ignoring_missed_data_line)))
self.settings['plotter']['series_linewidth'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_linewidth', error_ignoring = True, default_param_value = series_linewidth)))
self.settings['plotter']['series_marker'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series_marker', error_ignoring = True, default_param_value = series_marker))
self.settings['plotter']['series_markersize'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_markersize', error_ignoring = True, default_param_value = series_markersize)))
self.settings['plotter']['series_color'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series_color', error_ignoring = True, default_param_value = series_color))
self.settings['plotter']['series_alpha'] = tools.float_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_alpha', error_ignoring = True, default_param_value = series_alpha)))
#check several params
def check_elements_number(section1, param1, section2, param2):
if self.errors.error_occured:
return None
elements_number1 = len(self.settings[section1][param1])
elements_number2 = self.settings[section2][param2]
if elements_number1 != elements_number2:
self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] value(' + str(elements_number2) + ')')
check_elements_number('plotter', 'subplot_height_share', 'plotter', 'subplots_number')
def compare_elements_number(section1, param1, section2, param2):
if self.errors.error_occured:
return None
elements_number1 = len(self.settings[section1][param1])
elements_number2 = len(self.settings[section2][param2])
if elements_number1 != elements_number2:
self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] elements number(' + str(elements_number2) + ')')
compare_elements_number(input_feed_format, 'columns', input_feed_format, 'column_data_types')
compare_elements_number('plotter', 'x_ticks_data_columns', 'plotter', 'x_ticks')
compare_elements_number('plotter', 'x_ticks_data_columns', 'plotter', 'x_labels_format')
compare_elements_number('plotter', 'series', 'plotter', 'seria_to_subbplot_binding')
def main(self, args):
self.read_settings(args)
# print(self.settings)
fs = FileSystem(self.errors)
data_stream = DataStream(self.errors)
dp = DataProccessing(self.errors)
tools = Tools(self.errors)
input_file_path = self.settings['input']['file_path']
input_feed_format = self.settings[self.settings['input']
['input_feed_format']]
moex_currency_file = self.settings['input']['moex_currency_file']
moex_currency_feed_format = self.settings[
self.settings['input']['moex_currency_feed_format']]
output_folder = self.settings['output']['folder']
output_file = self.settings['output']['file']
output_file_path = output_folder + output_file
output_feed_format = self.settings[self.settings['output']
['output_feed_format']]
fs.create_folder_branch(output_folder)
data_stream.open_stream(input_file_path, input_feed_format)
data = data_stream.read_all(input_feed_format)
data_stream.close_stream()
data_stream.open_stream(moex_currency_file, moex_currency_feed_format)
moex_currency_data = data_stream.read_all(moex_currency_feed_format)
data_stream.close_stream()
time_range = dp.generate_time_range(
'12:00:00', '00:00:00', '00:05:00',
tools.explode(',', '18:50:00-19:05:00'))
date_range = dp.select_date_range(moex_currency_data['<DATE>'])
moex_currency_timed_data = dp.create_data_by_time_range(
time_range, date_range, moex_currency_data, ['<USDRUR>'])
# print(moex_currency_timed_data['<USDRUR>'])
dp.add_column('<USDRUR>', 'num', len(data['<DATE>']), data,
output_feed_format)
di_moex_timed_data = DataIterator(self.errors,
moex_currency_timed_data, '<DATE>',
moex_currency_feed_format)
di_data = DataIterator(self.errors, data, '<DATE>', output_feed_format)
last_USDRUR = None
while not di_data.EOD:
d_rec = di_data.get_next_rec()
while not di_moex_timed_data.EOD:
mtd_rec = di_moex_timed_data.get_next_rec()
if mtd_rec['<USDRUR>'] != None:
last_USDRUR = mtd_rec['<USDRUR>']
if d_rec['<DATE>']['yyyymmdd'] > mtd_rec['<DATE>']:
continue
else:
if d_rec['<DATE>']['yyyymmdd'] == mtd_rec[
'<DATE>'] and d_rec['<TIME>']['hhmmss'] > mtd_rec[
'<TIME>']:
continue
else:
if d_rec['<DATE>']['yyyymmdd'] == mtd_rec['<DATE>']:
data['<USDRUR>'][
di_data.rec_cnt -
1] = last_USDRUR #mtd_rec['<USDRUR>']
if d_rec['<TIME>']['hhmmss'] > mtd_rec['<TIME>']:
di_moex_timed_data.rec_cnt -= 1
break
else:
continue
# print(data['<USDRUR>'])
# print(len(data['<USDRUR>']))
# print(len(data['<DATE>']))
# di_data = DataIterator(self.errors, data, '<DATE>', output_feed_format)
# while not di_data.EOD:
# d_rec = di_data.get_next_rec()
# print(d_rec['<DATE>']['yyyymmdd'] + ' ' + d_rec['<TIME>']['hhmmss'] + ' ' + str(d_rec['<USDRUR>']))
dp.add_column('<Si_RUR>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Eu_RUR>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<ED_RUR>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<gamma>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<gamma_avg>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Si_open_long>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Si_open_short>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Si_lots>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Si_close_long>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Si_close_short>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Eu_open_long>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Eu_open_short>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Eu_lots>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Eu_close_long>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Eu_close_short>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<ED_open_long>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<ED_open_short>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<ED_lots>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<ED_close_long>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<ED_close_short>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Eu_eqv>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<Si_eqv>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<ED_eqv>', 'num', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<open_time>', 'str', len(data['<DATE>']), data,
output_feed_format)
dp.add_column('<all_eqv>', 'num', len(data['<DATE>']), data,
output_feed_format)
avg_per = 21
sma = SMA(avg_per)
N = 7
for rec_cnt in range(len(data['<DATE>'])):
rec = dp.get_rec(rec_cnt, data, output_feed_format)
Si_C = rec['<Si_C>']
Eu_C = rec['<Eu_C>']
ED_C = rec['<ED_C>']
USDRUR = rec['<USDRUR>']
if rec_cnt > 0:
Si_RUR += Si_C - last_Si_C
Eu_RUR += Eu_C - last_Eu_C
ED_RUR += (ED_C - last_ED_C) * Si_C #USDRUR * 1000
gamma = (Eu_RUR - Si_RUR - ED_RUR
) * N - Si_RUR # N*Eu_RUR - (N+1)*Si_RUR - N*ED_RUR
gamma_avg = sma.calc(gamma)
self.traiding(gamma, gamma_avg, rec, N)
# print(ED_RUR)
else:
Si_RUR = 0
Eu_RUR = 0
ED_RUR = 0
gamma = 0
gamma_avg = None
last_Si_C = Si_C
last_Eu_C = Eu_C
last_ED_C = ED_C
data['<Si_RUR>'][rec_cnt] = Si_RUR
data['<Eu_RUR>'][rec_cnt] = Eu_RUR
data['<ED_RUR>'][rec_cnt] = ED_RUR
data['<gamma>'][rec_cnt] = gamma
data['<gamma_avg>'][rec_cnt] = gamma_avg
data['<Si_open_long>'][rec_cnt] = rec['<Si_open_long>']
data['<Si_open_short>'][rec_cnt] = rec['<Si_open_short>']
data['<Si_lots>'][rec_cnt] = rec['<Si_lots>']
data['<Si_close_long>'][rec_cnt] = rec['<Si_close_long>']
data['<Si_close_short>'][rec_cnt] = rec['<Si_close_short>']
data['<Eu_open_long>'][rec_cnt] = rec['<Eu_open_long>']
data['<Eu_open_short>'][rec_cnt] = rec['<Eu_open_short>']
data['<Eu_lots>'][rec_cnt] = rec['<Eu_lots>']
data['<Eu_close_long>'][rec_cnt] = rec['<Eu_close_long>']
data['<Eu_close_short>'][rec_cnt] = rec['<Eu_close_short>']
data['<ED_open_long>'][rec_cnt] = rec['<ED_open_long>']
data['<ED_open_short>'][rec_cnt] = rec['<ED_open_short>']
data['<ED_lots>'][rec_cnt] = rec['<ED_lots>']
data['<ED_close_long>'][rec_cnt] = rec['<ED_close_long>']
data['<ED_close_short>'][rec_cnt] = rec['<ED_close_short>']
data['<Eu_eqv>'][rec_cnt] = rec['<Eu_eqv>']
data['<Si_eqv>'][rec_cnt] = rec['<Si_eqv>']
data['<ED_eqv>'][rec_cnt] = rec['<ED_eqv>']
data['<open_time>'][rec_cnt] = rec['<open_time>']
data['<all_eqv>'][rec_cnt] = rec['<all_eqv>']
data_stream.open_stream(output_file_path, output_feed_format, mode='w')
data_stream.write_all(data, output_feed_format)
data_stream.close_stream()
# fig_name = '0000'
# plotter = Plotter(self.errors)
# plotter.plot_series(data, self.settings, fig_name)
print(self.deal_cnt)
if self.errors.error_occured:
self.errors.print_errors()
else:
print('OK\n')
def __init__(self, errors):
self.errors = errors
self.settings = {}
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.tools = Tools(self.errors)
class SettingsReader:
def __init__(self, errors):
self.errors = errors
self.settings = {}
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.tools = Tools(self.errors)
def check_elements_number(self, settings, section1, param1, section2, param2):
elements_number1 = len(settings[section1][param1])
elements_number2 = settings[section2][param2]
if elements_number1 != elements_number2:
self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] value(' + str(elements_number2) + ')')
def compare_elements_number(self, settings, section1, param1, section2, param2):
elements_number1 = len(settings[section1][param1])
elements_number2 = len(settings[section2][param2])
if elements_number1 != elements_number2:
self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] elements number(' + str(elements_number2) + ')')
def clone_value(self, value, number):
return number * [value]
def read_ArbitrageSettings(self, settings, ini_file_path, encoding):
if self.errors.error_occured:
return None
self.ini_parser.read_ini(ini_file_path, encoding)
settings['input'] = {}
settings['input']['file_path'] = self.ini_parser.get_param('input', 'file_path')
settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
settings['input']['moex_currency_file'] = self.ini_parser.get_param('input', 'moex_currency_file')
settings['input']['moex_currency_feed_format'] = self.ini_parser.get_param('input', 'moex_currency_feed_format')
settings['output'] = {}
settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
settings['output']['file'] = self.ini_parser.get_param('output', 'file')
settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
input_feed_format = settings['input']['input_feed_format']
settings[input_feed_format] = {}
settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
moex_currency_feed_format = settings['input']['moex_currency_feed_format']
settings[moex_currency_feed_format] = {}
settings[moex_currency_feed_format]['encoding'] = self.ini_parser.get_param(moex_currency_feed_format, 'encoding')
settings[moex_currency_feed_format]['header_lines_number'] = self.ini_parser.get_param(moex_currency_feed_format, 'header_lines_number', 'int')
settings[moex_currency_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(moex_currency_feed_format, 'columns'))
settings[moex_currency_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(moex_currency_feed_format, 'column_separator'))
settings[moex_currency_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(moex_currency_feed_format, 'column_data_types'))
output_feed_format = settings['output']['output_feed_format']
settings[output_feed_format] = {}
settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
#check several params
self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
self.compare_elements_number(settings, moex_currency_feed_format, 'columns', moex_currency_feed_format, 'column_data_types')
self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
def read_JoinData_settings(self, settings, ini_file_path, encoding):
if self.errors.error_occured:
return None
self.ini_parser.read_ini(ini_file_path, encoding)
settings['input'] = {}
settings['input']['folder'] = self.ini_parser.get_param('input', 'folder')
settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
settings['input']['columns'] = self.tools.explode(',', self.ini_parser.get_param('input', 'columns'))
settings['input']['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param('input', 'column_data_types'))
settings['input']['date_col'] = self.ini_parser.get_param('input', 'date_col')
settings['input']['start_time'] = self.ini_parser.get_param('input', 'start_time')
settings['input']['stop_time'] = self.ini_parser.get_param('input', 'stop_time')
settings['input']['step_time'] = self.ini_parser.get_param('input', 'step_time')
settings['input']['exclude_time'] = self.tools.explode(',', self.ini_parser.get_param('input', 'exclude_time'))
settings['output'] = {}
settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
settings['output']['file'] = self.ini_parser.get_param('output', 'file')
settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
input_feed_format = settings['input']['input_feed_format']
settings[input_feed_format] = {}
settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
output_feed_format = settings['output']['output_feed_format']
settings[output_feed_format] = {}
settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
# settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
# settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
#check several params
self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
# self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
self.compare_elements_number(settings, 'input', 'columns', 'input', 'column_data_types')
def read_ConcatinateData_settings(self, settings, ini_file_path, encoding):
if self.errors.error_occured:
return None
self.ini_parser.read_ini(ini_file_path, encoding)
settings['input'] = {}
settings['input']['folder'] = self.ini_parser.get_param('input', 'folder')
settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
settings['output'] = {}
settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
settings['output']['file'] = self.ini_parser.get_param('output', 'file')
settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
input_feed_format = settings['input']['input_feed_format']
settings[input_feed_format] = {}
settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
output_feed_format = settings['output']['output_feed_format']
settings[output_feed_format] = {}
settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
#check several params
self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
def read_PlotterSettings(self, settings, ini_file_path, encoding):
if self.errors.error_occured:
return None
settings['output'] = {}
settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
settings['plotter'] = {}
settings['plotter']['fig_folder'] = settings['output']['folder'] + self.ini_parser.get_param('plotter', 'fig_folder')
settings['plotter']['x_ticks_data_columns'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks_data_columns'))
settings['plotter']['x_ticks'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks'))
settings['plotter']['x_labels_format'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_labels_format'))
settings['plotter']['series'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series'))
settings['plotter']['subplot_height_share'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'subplot_height_share')))
settings['plotter']['subplots_number'] = self.ini_parser.get_param('plotter', 'subplots_number', 'int')
settings['plotter']['seria_to_subbplot_binding'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'seria_to_subbplot_binding')))
ignoring_missed_data_line = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'ignoring_missed_data_line'), len(settings['plotter']['series'])))
series_linewidth = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_linewidth'), len(settings['plotter']['series'])))
series_markersize = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_markersize'), len(settings['plotter']['series'])))
series_marker = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_marker'), len(settings['plotter']['series'])))
series_color = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_color'), len(settings['plotter']['series'])))
series_alpha = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_alpha'), len(settings['plotter']['series'])))
settings['plotter']['ignoring_missed_data_line'] = self.tools.bool_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'ignoring_missed_data_line', error_ignoring = True, default_param_value = ignoring_missed_data_line)))
settings['plotter']['series_linewidth'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_linewidth', error_ignoring = True, default_param_value = series_linewidth)))
settings['plotter']['series_marker'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_marker', error_ignoring = True, default_param_value = series_marker))
settings['plotter']['series_markersize'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_markersize', error_ignoring = True, default_param_value = series_markersize)))
settings['plotter']['series_color'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_color', error_ignoring = True, default_param_value = series_color))
settings['plotter']['series_alpha'] = self.tools.float_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_alpha', error_ignoring = True, default_param_value = series_alpha)))
#check several params
self.check_elements_number(settings, 'plotter', 'subplot_height_share', 'plotter', 'subplots_number')
self.compare_elements_number(settings, 'plotter', 'x_ticks_data_columns', 'plotter', 'x_ticks')
self.compare_elements_number(settings, 'plotter', 'x_ticks_data_columns', 'plotter', 'x_labels_format')
self.compare_elements_number(settings, 'plotter', 'series', 'plotter', 'seria_to_subbplot_binding')
def __init__(self, errors):
self._errors = errors
self._tools = Tools(self._errors)
self._ini_file_path = ''
self.settings = {}
def main(self, args):
self.read_settings(args)
# print(self.settings)
fs = FileSystem(self.errors)
data_stream = DataStream(self.errors)
dp = DataProccessing(self.errors)
tools = Tools(self.errors)
input_folder = self.settings['input']['folder']
input_feed_format = self.settings[self.settings['input']
['input_feed_format']]
input_columns = self.settings['input']['columns']
input_column_data_types = self.settings['input']['column_data_types']
output_folder = self.settings['output']['folder']
output_file = self.settings['output']['file']
output_file_path = output_folder + output_file
output_feed_format = self.settings[self.settings['output']
['output_feed_format']]
input_date_col = self.settings['input']['date_col']
start_time = self.settings['input']['start_time']
stop_time = self.settings['input']['stop_time']
step_time = self.settings['input']['step_time']
exclude_time = self.settings['input']['exclude_time']
# print(input_columns);
folder_list = fs.get_folder_list(input_folder)
folder_list.sort()
file_cnt = 0
files_number = len(folder_list)
joined_data = {}
timed_data = {}
joined_data['<DATE>'] = []
joined_data['<TIME>'] = []
output_feed_format['columns'] = ['<DATE>', '<TIME>']
output_feed_format['column_data_types'] = ['yyyymmdd', 'hhmmss']
for file_cnt in range(files_number):
for col_cnt in range(len(input_columns)):
joined_data[input_columns[col_cnt] + '_' +
str(file_cnt + 1)] = []
output_feed_format['columns'].append(input_columns[col_cnt] +
'_' + str(file_cnt + 1))
output_feed_format['column_data_types'].append(
input_column_data_types[col_cnt])
date_range = []
file_cnt = 0
while not self.errors.error_occured and file_cnt < files_number:
input_file_path = input_folder + folder_list[file_cnt]
data_stream.open_stream(input_file_path, input_feed_format)
data = data_stream.read_all(input_feed_format)
data_stream.close_stream()
if file_cnt == 0:
time_range = dp.generate_time_range(start_time, stop_time,
step_time, exclude_time)
date_range = dp.select_date_range(data[input_date_col])
timed_data[file_cnt] = dp.create_data_by_time_range(
time_range, date_range, data, input_columns)
file_cnt += 1
rec_cnt = 0
for c_date in date_range:
if self.errors.error_occured:
break
for c_time in time_range:
rec = {}
exitFlag = False
for file_cnt in range(files_number):
for col in input_columns:
if timed_data[file_cnt][col][rec_cnt] != None:
rec[col + '_' +
str(file_cnt +
1)] = timed_data[file_cnt][col][rec_cnt]
else:
rec = {}
exitFlag = True
break
if exitFlag:
break
if len(rec) > 0:
joined_data['<DATE>'].append(c_date)
joined_data['<TIME>'].append(c_time)
for k in rec:
joined_data[k].append(rec[k])
rec_cnt += 1
data_stream.open_stream(output_file_path, output_feed_format, mode='w')
data_stream.write_all(joined_data, output_feed_format)
data_stream.close_stream()
if self.errors.error_occured:
self.errors.print_errors()
else:
print('OK\n')