class Parser:
def __init__(self):
self.errors = Errors()
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.settings = {}
def read_settings(self, args):
if len(args) < 2:
self.errors.raise_error('No ini file path')
else:
encoding = self.ini_encoding
if len(args) > 2:
encoding = args[2]
ini_file_path = args[1]
self.ini_parser.read_ini(args[1], encoding)
def set_params(self, args):
self.read_settings(args)
self.settings['input'] = {}
self.settings['output'] = {}
self.settings['params'] = {}
self.settings['input']['file_path'] = self.ini_parser.get_param(
'input', 'file_path')
self.settings['input']['encoding'] = self.ini_parser.get_param(
'input', 'encoding')
self.settings['params'][
'word_common_separators'] = self.ini_parser.get_param(
'params',
'word_common_separators',
param_type='str_array',
sep='\\')
self.settings['params'][
'start_subsentence'] = self.ini_parser.get_param(
'params', 'start_subsentence', param_type='str_array')
self.settings['params']['end_subsentence'] = self.ini_parser.get_param(
'params', 'end_subsentence', param_type='str_array', sep='\\')
self.settings['params']['end_sentence'] = self.ini_parser.get_param(
'params', 'end_sentence', param_type='str_array', sep='\\')
def main(self, args):
self.set_params(args)
# print(self.settings)
text_parser = TextParser(self.errors)
# input_file_name = self.settings['input']['file_path']
# enc = self.settings['input']['encoding']
# params = self.settings['params']
text_parser.parse_text(self.settings)
if self.errors.error_occured:
self.errors.print_errors()
else:
print('OK\n')
class Template:
def __init__(self):
self.errors = Errors()
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.settings = {}
def read_settings(self, args):
if len(args) < 2:
self.errors.raise_error('no ini file path')
else:
encoding = self.ini_encoding
if len(args) > 2:
encoding = args[2]
ini_file_path = args[1]
self.ini_parser.read_ini(args[1], encoding)
def set_params(self, args):
tools = Tools(self.errors)
self.read_settings(args)
self.settings['g1'] = self.ini_parser.get_param('g1', None)
self.settings['g2'] = {}
self.settings['g2']['p1'] = tools.explode(
',', self.ini_parser.get_param('g2', 'p1'))
self.settings['g3'] = {}
self.settings['g3']['i4'] = self.ini_parser.get_param(
'g3', 'i4', 'int')
def main(self, args):
self.set_params(args)
print(self.settings)
if self.errors.error_occured:
self.errors.print_errors()
else:
print('OK\n')
class Template:
def __init__(self):
self.errors = Errors()
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.settings = {}
def read_settings(self, args):
if len(args) < 2:
self.errors.raise_error('No ini file path')
else:
encoding = self.ini_encoding
if len(args) > 2:
encoding = args[2]
ini_file_path = args[1]
self.ini_parser.read_ini(args[1], encoding)
def set_params(self, args):
self.read_settings(args)
self.settings['g2'] = {}
self.settings['g3'] = {}
self.settings['g2']['p1'] = self.ini_parser.get_param(
'g2', 'p1', 'str_array')
self.settings['g3'] = self.ini_parser.get_param('g3')
def main(self, args):
self.set_params(args)
print(self.settings)
g1_p2 = self.ini_parser.get_param('g1', 'p2')
print(g1_p2)
if self.errors.error_occured:
self.errors.print_errors()
else:
print('OK\n')
def __init__(self):
self.errors = Errors()
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.settings = {}
class GetFinamData:
def __init__(self):
self.errors = Errors()
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.tc_ini_parser = IniParser(self.errors)
self.settings = {}
def read_settings(self, args):
if len(args) < 2:
self.errors.raise_error('no ini file path')
else:
encoding = self.ini_encoding
if len(args) > 2:
self.ini_encoding = args[2]
ini_file_path = args[1]
self.ini_parser.read_ini(args[1], self.ini_encoding)
def set_params(self, args):
tools = Tools(self.errors)
self.read_settings(args)
self.settings['common'] = {}
self.settings['common']['time_frames'] = tools.explode(
',', self.ini_parser.get_param('common', 'time_frames'))
self.settings['common']['output_folder'] = self.ini_parser.get_param(
'common', 'output_folder')
self.settings['common'][
'trading_calendar'] = self.ini_parser.get_param(
'common', 'trading_calendar')
self.settings['common'][
'always_update_past_working_days_number'] = self.ini_parser.get_param(
'common', 'always_update_past_working_days_number', 'int')
self.tc_ini_parser.read_ini(
self.settings['common']['trading_calendar'], self.ini_encoding)
# self.settings['non_working_days'] = {}
# self.settings['non_working_days'] = self.tc_ini_parser.get_param('non_working_days')
self.settings['contracts'] = {}
tickers = tools.explode(
',', self.ini_parser.get_param('contracts', 'tickers'))
for ticker_idx in range(len(tickers)):
self.settings['contracts'][ticker_idx] = {}
self.settings['contracts'][ticker_idx]['ticker'] = tickers[
ticker_idx]
self.settings['contracts'][ticker_idx]['list'] = {}
list = tools.explode(
',', self.ini_parser.get_param('contracts',
tickers[ticker_idx]))
for contract_idx in range(len(list)):
self.settings['contracts'][ticker_idx]['list'][
contract_idx] = self.ini_parser.get_param(
list[contract_idx])
def get_contract(self, _idx=[0, 0]):
if self.errors.error_occured:
return None, None, None, None, None, None
if _idx[0] < len(self.settings['contracts']):
if _idx[1] < len(self.settings['contracts'][_idx[0]]['list']):
Ticker = self.settings['contracts'][_idx[0]]['ticker']
ContractSymbol = self.settings['contracts'][_idx[0]]['list'][
_idx[1]]['ContractSymbol']
ContractTradingSymbol = self.settings['contracts'][
_idx[0]]['list'][_idx[1]]['ContractTradingSymbol']
FirstTradingDay = self.settings['contracts'][_idx[0]]['list'][
_idx[1]]['FirstTradingDay']
LastTradingDay = self.settings['contracts'][_idx[0]]['list'][
_idx[1]]['LastTradingDay']
FinamEm = self.settings['contracts'][_idx[0]]['list'][
_idx[1]]['FinamEm']
_idx[1] += 1
else:
_idx[0] += 1
_idx[1] = 0
Ticker, ContractSymbol, ContractTradingSymbol, FirstTradingDay, LastTradingDay, FinamEm = self.get_contract(
_idx)
else:
return None, None, None, None, None, None
return Ticker, ContractSymbol, ContractTradingSymbol, FirstTradingDay, LastTradingDay, FinamEm
def shape_finam_url(self, current_trading_day, arch, FinamEm,
ContractSymbol, time_frame):
# url = http://export.finam.ru/SPFB.Eu-3.19_190108_190108.txt?
# market = 14 # Номер рынка
# em = 487593 # Номер инструмента
# code = SPFB.Eu-3.19 # Тикер инструмента
# apply = 0 #
# df = 8 # Начальная дата, номер дня (1-31)
# mf = 0 # Начальная дата, номер месяца (0-11)
# yf = 2019 # Начальная дата, год
# from = 08.01.2019 # Начальная дата
# dt = 8 # Конечная дата, номер дня
# mt = 0 # Конечная дата, номер месяца
# yt = 2019 # Конечная дата, год
# to = 08.01.2019 # Конечная дата
# p = 3 # Таймфрейм {'tick': 1, 'min': 2, '5min': 3, '10min': 4, '15min': 5, '30min': 6, 'hour': 7, 'daily': 8, 'week': 9, 'month': 10}
# f = SPFB.Eu-3.19_190108_190108 # Имя сформированного файла
# e = .txt # Расширение сформированного файла# возможны варианты — .txt либо .csv
# cn = SPFB.Eu-3.19 # Имя контракта
# dtf = 1 # формат даты (1 — ггггммдд, 2 — ггммдд, 3 — ддммгг, 4 — дд/мм/гг, 5 — мм/дд/гг)
# tmf = 1 # формат времени (1 — ччммсс, 2 — ччмм, 3 — чч: мм: сс, 4 — чч: мм)
# MSOR = 1 # выдавать время (0 — начала свечи, 1 — окончания свечи)
# mstime = on # выдавать время (НЕ московское — mstimever=0# московское — mstime='on', mstimever='1')
# mstimever = 1 # Коррекция часового пояса
# sep = 1 # Разделитель полей (1 — запятая (,), 2 — точка (.), 3 — точка с запятой (;), 4 — табуляция (»), 5 — пробел ( ))
# sep2 = 1 # Разделитель разрядов (1 — нет, 2 — точка (.), 3 — запятая (,), 4 — пробел ( ), 5 — кавычка ('))
# datf = 1 # Перечень получаемых данных (#1 — TICKER, PER, DATE, TIME, OPEN, HIGH, LOW, CLOSE, VOL# #2 — TICKER, PER, DATE, TIME, OPEN, HIGH, LOW, CLOSE# #3 — TICKER, PER, DATE, TIME, CLOSE, VOL# #4 — TICKER, PER, DATE, TIME, CLOSE# #5 — DATE, TIME, OPEN, HIGH, LOW, CLOSE, VOL# #6 — DATE, TIME, LAST, VOL, ID, OPER).
# at = 1 # добавлять заголовок в файл (0 — нет, 1 — да)
url = 'http://export.finam.ru/'
if arch:
market = '13'
else:
market = '14'
em = FinamEm
code = 'SPFB.' + ContractSymbol
apply = '0'
df = str(current_trading_day.day)
mf = str(current_trading_day.month - 1)
yf = str(current_trading_day.year)
_from = dt.strftime(current_trading_day, '%d.%m.%Y')
_dt = str(current_trading_day.day)
mt = str(current_trading_day.month - 1)
yt = str(current_trading_day.year)
to = dt.strftime(current_trading_day, '%d.%m.%Y')
time_frame_codes = {
'0': '1',
'1': '2',
'5': '3',
'10': '4',
'15': '5',
'30': '6',
'60': '7',
'D': '8',
'W': '9',
'M': '10'
}
p = time_frame_codes[time_frame]
f = code + '_' + dt.strftime(current_trading_day, '%Y-%m-%d')
e = '.txt'
cn = code
dtf = '1'
tmf = '1'
MSOR = '1'
mstime = 'on'
mstimever = '1'
sep = '1'
sep2 = '1'
datf = '1'
at = '1'
url += (f + e + '?market=' + market + '&em=' + em + '&code=' + code +
'&apply=' + apply + '&df=' + df + '&mf=' + mf + '&yf=' + yf +
'&from=' + _from + '&dt=' + _dt + '&mt=' + mt + '&yt=' + yt +
'&to=' + to + '&p=' + p + '&f=' + f + '&e=' + e + '&cn=' + cn +
'&dtf=' + dtf + '&tmf=' + tmf + '&MSOR=' + MSOR + '&mstime=' +
mstime + '&mstimever=' + mstimever + '&sep=' + sep + '&sep2=' +
sep2 + '&datf=' + datf + '&at=' + at + '')
return url, f + e
def is_non_working_day(self, current_trading_day):
ctd_year = str(current_trading_day.year)
ctd_month = str(current_trading_day.month)
ctd_day = str(current_trading_day.day)
tools = Tools(self.errors)
nw_days = tools.explode(
',',
self.tc_ini_parser.get_param('non_working_days',
ctd_month + '.' + ctd_year))
if ctd_day in nw_days:
return True
return False
def allow_update(self, now_day, current_trading_day, file_path):
if self.errors.error_occured:
return False
day_cnt = 1
days_shift = datetime.timedelta(days=0)
always_update_past_working_days_number = self.settings['common'][
'always_update_past_working_days_number']
while always_update_past_working_days_number > 0:
days_shift = datetime.timedelta(days=day_cnt)
if not self.is_non_working_day(now_day - days_shift):
always_update_past_working_days_number -= 1
day_cnt += 1
if current_trading_day < now_day - days_shift:
if not self.is_non_working_day(current_trading_day):
if not os.path.exists(file_path):
return True
elif current_trading_day <= now_day:
if not self.is_non_working_day(current_trading_day):
return True
return False
def main(self, args):
self.set_params(args)
fs = FileSystem(self.errors)
now_day = dt.today().date()
time_frames = self.settings['common']['time_frames']
while not self.errors.error_occured:
Ticker, ContractSymbol, ContractTradingSymbol, FirstTradingDay, LastTradingDay, FinamEm = self.get_contract(
)
if Ticker:
first_trading_day = dt.strptime(FirstTradingDay,
'%d.%m.%Y').date()
last_trading_day = dt.strptime(LastTradingDay,
'%d.%m.%Y').date()
ltd_year = last_trading_day.year
# delta = last_trading_day - first_trading_day
one_day = datetime.timedelta(days=1)
current_trading_day = first_trading_day
while not self.errors.error_occured:
# for day_cnt in range(delta.days):
for time_frame in time_frames:
if now_day > last_trading_day:
arch = True
else:
arch = False
path = self.settings['common']['output_folder'] + str(
ltd_year
) + '/' + Ticker + '/' + ContractSymbol + '/' + time_frame + '/'
fs.create_folder_branch(path)
url, file = self.shape_finam_url(
current_trading_day, arch, FinamEm, ContractSymbol,
time_frame)
file_path = path + file
if self.allow_update(now_day, current_trading_day,
file_path):
print(Ticker, ContractSymbol, current_trading_day,
time_frame)
try:
page = urllib.request.urlopen(url)
except Exception as e:
self.errors.raise_error('Can\'t open url ' +
url)
break
content = page.read()
content = content.decode('utf-8').replace('\r', '')
try:
with open(file_path, "w") as text_file:
print(content, file=text_file)
except Exception as e:
self.errors.raise_error('Can\'t write file ' +
file_path)
break
time.sleep(1)
current_trading_day += one_day
if current_trading_day > last_trading_day:
break
else:
break
# print(self.settings)
if self.errors.error_occured:
self.errors.print_errors()
else:
print('OK\n')
class Plot:
def __init__(self):
self.errors = Errors()
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.settings = {}
def read_settings(self, args):
if len(args) < 2:
self.errors.raise_error('no ini file path')
else:
encoding = self.ini_encoding
if len(args) > 2:
encoding = args[2]
ini_file_path = args[1]
self.ini_parser.read_ini(args[1], encoding)
def set_params(self, args):
self.read_settings(args)
tools = Tools(self.errors)
self.settings['input'] = {}
self.settings['input']['file_path'] = self.ini_parser.get_param('input', 'file_path')
self.settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
self.settings['output'] = {}
self.settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
input_feed_format = self.settings['input']['input_feed_format']
self.settings[input_feed_format] = {}
self.settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
self.settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
self.settings[input_feed_format]['columns'] = tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
self.settings[input_feed_format]['column_separator'] = tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
self.settings[input_feed_format]['column_data_types'] = tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
self.settings['plotter'] = {}
self.settings['plotter']['fig_folder'] = self.settings['output']['folder'] + self.ini_parser.get_param('plotter', 'fig_folder')
self.settings['plotter']['x_ticks_data_columns'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks_data_columns'))
self.settings['plotter']['x_ticks'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks'))
self.settings['plotter']['x_labels_format'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_labels_format'))
self.settings['plotter']['series'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series'))
self.settings['plotter']['subplot_height_share'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'subplot_height_share')))
self.settings['plotter']['subplots_number'] = self.ini_parser.get_param('plotter', 'subplots_number', 'int')
self.settings['plotter']['seria_to_subbplot_binding'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'seria_to_subbplot_binding')))
def clone_value(value, number):
return number * [value]
ignoring_missed_data_line = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'ignoring_missed_data_line'), len(self.settings['plotter']['series'])))
series_linewidth = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_linewidth'), len(self.settings['plotter']['series'])))
series_markersize = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_markersize'), len(self.settings['plotter']['series'])))
series_marker = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_marker'), len(self.settings['plotter']['series'])))
series_color = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_color'), len(self.settings['plotter']['series'])))
series_alpha = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_alpha'), len(self.settings['plotter']['series'])))
self.settings['plotter']['ignoring_missed_data_line'] = tools.bool_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'ignoring_missed_data_line', error_ignoring = True, default_param_value = ignoring_missed_data_line)))
self.settings['plotter']['series_linewidth'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_linewidth', error_ignoring = True, default_param_value = series_linewidth)))
self.settings['plotter']['series_marker'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series_marker', error_ignoring = True, default_param_value = series_marker))
self.settings['plotter']['series_markersize'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_markersize', error_ignoring = True, default_param_value = series_markersize)))
self.settings['plotter']['series_color'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series_color', error_ignoring = True, default_param_value = series_color))
self.settings['plotter']['series_alpha'] = tools.float_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_alpha', error_ignoring = True, default_param_value = series_alpha)))
#check several params
def check_elements_number(section1, param1, section2, param2):
if self.errors.error_occured:
return None
elements_number1 = len(self.settings[section1][param1])
elements_number2 = self.settings[section2][param2]
if elements_number1 != elements_number2:
self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] value(' + str(elements_number2) + ')')
check_elements_number('plotter', 'subplot_height_share', 'plotter', 'subplots_number')
def compare_elements_number(section1, param1, section2, param2):
if self.errors.error_occured:
return None
elements_number1 = len(self.settings[section1][param1])
elements_number2 = len(self.settings[section2][param2])
if elements_number1 != elements_number2:
self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] elements number(' + str(elements_number2) + ')')
compare_elements_number(input_feed_format, 'columns', input_feed_format, 'column_data_types')
compare_elements_number('plotter', 'x_ticks_data_columns', 'plotter', 'x_ticks')
compare_elements_number('plotter', 'x_ticks_data_columns', 'plotter', 'x_labels_format')
compare_elements_number('plotter', 'series', 'plotter', 'seria_to_subbplot_binding')
def main(self, args):
self.set_params(args)
data_stream = DataStream(self.errors)
input_file_path = self.settings['input']['file_path']
input_feed_format = self.settings[self.settings['input']['input_feed_format']]
data_stream.open_stream(input_file_path, input_feed_format)
data = data_stream.read_all(input_feed_format)
data_stream.close_stream()
fig_name = '0000'
plotter = Plotter(self.errors)
plotter.plot_series(data, self.settings, fig_name)
# print(data)
if self.errors.error_occured:
self.errors.print_errors()
else:
print('OK\n')
def __init__(self, errors):
self.errors = errors
self.settings = {}
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.tools = Tools(self.errors)
class SettingsReader:
def __init__(self, errors):
self.errors = errors
self.settings = {}
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.tools = Tools(self.errors)
def check_elements_number(self, settings, section1, param1, section2, param2):
elements_number1 = len(settings[section1][param1])
elements_number2 = settings[section2][param2]
if elements_number1 != elements_number2:
self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] value(' + str(elements_number2) + ')')
def compare_elements_number(self, settings, section1, param1, section2, param2):
elements_number1 = len(settings[section1][param1])
elements_number2 = len(settings[section2][param2])
if elements_number1 != elements_number2:
self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] elements number(' + str(elements_number2) + ')')
def clone_value(self, value, number):
return number * [value]
def read_ArbitrageSettings(self, settings, ini_file_path, encoding):
if self.errors.error_occured:
return None
self.ini_parser.read_ini(ini_file_path, encoding)
settings['input'] = {}
settings['input']['file_path'] = self.ini_parser.get_param('input', 'file_path')
settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
settings['input']['moex_currency_file'] = self.ini_parser.get_param('input', 'moex_currency_file')
settings['input']['moex_currency_feed_format'] = self.ini_parser.get_param('input', 'moex_currency_feed_format')
settings['output'] = {}
settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
settings['output']['file'] = self.ini_parser.get_param('output', 'file')
settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
input_feed_format = settings['input']['input_feed_format']
settings[input_feed_format] = {}
settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
moex_currency_feed_format = settings['input']['moex_currency_feed_format']
settings[moex_currency_feed_format] = {}
settings[moex_currency_feed_format]['encoding'] = self.ini_parser.get_param(moex_currency_feed_format, 'encoding')
settings[moex_currency_feed_format]['header_lines_number'] = self.ini_parser.get_param(moex_currency_feed_format, 'header_lines_number', 'int')
settings[moex_currency_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(moex_currency_feed_format, 'columns'))
settings[moex_currency_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(moex_currency_feed_format, 'column_separator'))
settings[moex_currency_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(moex_currency_feed_format, 'column_data_types'))
output_feed_format = settings['output']['output_feed_format']
settings[output_feed_format] = {}
settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
#check several params
self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
self.compare_elements_number(settings, moex_currency_feed_format, 'columns', moex_currency_feed_format, 'column_data_types')
self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
def read_JoinData_settings(self, settings, ini_file_path, encoding):
if self.errors.error_occured:
return None
self.ini_parser.read_ini(ini_file_path, encoding)
settings['input'] = {}
settings['input']['folder'] = self.ini_parser.get_param('input', 'folder')
settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
settings['input']['columns'] = self.tools.explode(',', self.ini_parser.get_param('input', 'columns'))
settings['input']['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param('input', 'column_data_types'))
settings['input']['date_col'] = self.ini_parser.get_param('input', 'date_col')
settings['input']['start_time'] = self.ini_parser.get_param('input', 'start_time')
settings['input']['stop_time'] = self.ini_parser.get_param('input', 'stop_time')
settings['input']['step_time'] = self.ini_parser.get_param('input', 'step_time')
settings['input']['exclude_time'] = self.tools.explode(',', self.ini_parser.get_param('input', 'exclude_time'))
settings['output'] = {}
settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
settings['output']['file'] = self.ini_parser.get_param('output', 'file')
settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
input_feed_format = settings['input']['input_feed_format']
settings[input_feed_format] = {}
settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
output_feed_format = settings['output']['output_feed_format']
settings[output_feed_format] = {}
settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
# settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
# settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
#check several params
self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
# self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
self.compare_elements_number(settings, 'input', 'columns', 'input', 'column_data_types')
def read_ConcatinateData_settings(self, settings, ini_file_path, encoding):
if self.errors.error_occured:
return None
self.ini_parser.read_ini(ini_file_path, encoding)
settings['input'] = {}
settings['input']['folder'] = self.ini_parser.get_param('input', 'folder')
settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
settings['output'] = {}
settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
settings['output']['file'] = self.ini_parser.get_param('output', 'file')
settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
input_feed_format = settings['input']['input_feed_format']
settings[input_feed_format] = {}
settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
output_feed_format = settings['output']['output_feed_format']
settings[output_feed_format] = {}
settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
#check several params
self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
def read_PlotterSettings(self, settings, ini_file_path, encoding):
if self.errors.error_occured:
return None
settings['output'] = {}
settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
settings['plotter'] = {}
settings['plotter']['fig_folder'] = settings['output']['folder'] + self.ini_parser.get_param('plotter', 'fig_folder')
settings['plotter']['x_ticks_data_columns'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks_data_columns'))
settings['plotter']['x_ticks'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks'))
settings['plotter']['x_labels_format'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_labels_format'))
settings['plotter']['series'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series'))
settings['plotter']['subplot_height_share'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'subplot_height_share')))
settings['plotter']['subplots_number'] = self.ini_parser.get_param('plotter', 'subplots_number', 'int')
settings['plotter']['seria_to_subbplot_binding'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'seria_to_subbplot_binding')))
ignoring_missed_data_line = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'ignoring_missed_data_line'), len(settings['plotter']['series'])))
series_linewidth = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_linewidth'), len(settings['plotter']['series'])))
series_markersize = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_markersize'), len(settings['plotter']['series'])))
series_marker = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_marker'), len(settings['plotter']['series'])))
series_color = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_color'), len(settings['plotter']['series'])))
series_alpha = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_alpha'), len(settings['plotter']['series'])))
settings['plotter']['ignoring_missed_data_line'] = self.tools.bool_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'ignoring_missed_data_line', error_ignoring = True, default_param_value = ignoring_missed_data_line)))
settings['plotter']['series_linewidth'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_linewidth', error_ignoring = True, default_param_value = series_linewidth)))
settings['plotter']['series_marker'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_marker', error_ignoring = True, default_param_value = series_marker))
settings['plotter']['series_markersize'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_markersize', error_ignoring = True, default_param_value = series_markersize)))
settings['plotter']['series_color'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_color', error_ignoring = True, default_param_value = series_color))
settings['plotter']['series_alpha'] = self.tools.float_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_alpha', error_ignoring = True, default_param_value = series_alpha)))
#check several params
self.check_elements_number(settings, 'plotter', 'subplot_height_share', 'plotter', 'subplots_number')
self.compare_elements_number(settings, 'plotter', 'x_ticks_data_columns', 'plotter', 'x_ticks')
self.compare_elements_number(settings, 'plotter', 'x_ticks_data_columns', 'plotter', 'x_labels_format')
self.compare_elements_number(settings, 'plotter', 'series', 'plotter', 'seria_to_subbplot_binding')
class GetMoexData:
def __init__(self):
self.errors = Errors()
self.ini_encoding = 'utf-8'
self.ini_parser = IniParser(self.errors)
self.tc_ini_parser = IniParser(self.errors)
self.settings = {}
def read_settings(self, args):
if len(args) < 2:
self.errors.raise_error('no ini file path')
else:
encoding = self.ini_encoding
if len(args) > 2:
self.ini_encoding = args[2]
ini_file_path = args[1]
self.ini_parser.read_ini(args[1], self.ini_encoding)
def set_params(self, args):
self.read_settings(args)
self.settings['output'] = {}
self.settings['output']['file_path'] = self.ini_parser.get_param(
'output', 'file_path')
self.settings['period'] = {}
self.settings['period']['start_date'] = self.ini_parser.get_param(
'period', 'start_date')
def shape_moex_url(self, start_date, end_date):
url = 'https://www.moex.com/export/derivatives/currency-rate.aspx?'
url = (url + 'language=en' + '¤cy=USD/RUB' + '&moment_start=' +
start_date + '&moment_end=' + end_date)
# print(urllib.parse.quote(url))
return url #urllib.parse.quote(url)
def extract_rates(self, content):
saved_content = '<CURRENCY>,<DATE>,<TIME>,<RATE>\n'
rates = []
while True:
rate_moment_pos = content.find('rate moment')
value_pos = content.find('value')
end_str_pos = content.find('/>')
rate_moment = content[rate_moment_pos + 13:value_pos - 2]
value = content[value_pos + 7:end_str_pos - 2]
if rate_moment_pos == -1:
break
else:
moment_date = rate_moment.split(' ')[0]
monent_time = rate_moment.split(' ')[1]
moment_date = dt.strftime(dt.strptime(moment_date, '%Y-%m-%d'),
'%Y%m%d')
monent_time = dt.strftime(dt.strptime(monent_time, '%H:%M:%S'),
'%H%M%S')
rates.append('USDRUB' + ',' + moment_date + ',' + monent_time +
',' + value)
content = content[end_str_pos + 2:]
rates.sort()
saved_content += '\n'.join(rates)
return saved_content
def main(self, args):
self.set_params(args)
fs = FileSystem(self.errors)
file_path = self.settings['output']['file_path']
start_date = self.settings['period']['start_date']
if not self.errors.error_occured:
start_date = dt.strftime(dt.strptime(start_date, '%d.%m.%Y'),
'%Y-%m-%d')
end_date = dt.strftime((dt.today()), '%Y-%m-%d')
url = self.shape_moex_url(start_date, end_date)
content = ''
try:
page = urllib.request.urlopen(url)
content = page.read()
content = content.decode('utf-8').replace('\r', '')
except Exception as e:
self.errors.raise_error('Can\'t open url ' + url + '\n' +
str(e))
content = self.extract_rates(content)
folder_path, file_name = fs.split_file_path(file_path)
fs.create_folder_branch(folder_path)
try:
with open(file_path, "w") as text_file:
print(content, file=text_file)
except Exception as e:
self.errors.raise_error('Can\'t write file ' + file_path)
if self.errors.error_occured:
self.errors.print_errors()
else:
print('OK\n')