Ejemplo n.º 1
0
class Parser:
    def __init__(self):
        self.errors = Errors()
        self.ini_encoding = 'utf-8'
        self.ini_parser = IniParser(self.errors)
        self.settings = {}

    def read_settings(self, args):
        if len(args) < 2:
            self.errors.raise_error('No ini file path')
        else:
            encoding = self.ini_encoding
            if len(args) > 2:
                encoding = args[2]

            ini_file_path = args[1]
            self.ini_parser.read_ini(args[1], encoding)

    def set_params(self, args):
        self.read_settings(args)
        self.settings['input'] = {}
        self.settings['output'] = {}
        self.settings['params'] = {}
        self.settings['input']['file_path'] = self.ini_parser.get_param(
            'input', 'file_path')
        self.settings['input']['encoding'] = self.ini_parser.get_param(
            'input', 'encoding')
        self.settings['params'][
            'word_common_separators'] = self.ini_parser.get_param(
                'params',
                'word_common_separators',
                param_type='str_array',
                sep='\\')
        self.settings['params'][
            'start_subsentence'] = self.ini_parser.get_param(
                'params', 'start_subsentence', param_type='str_array')
        self.settings['params']['end_subsentence'] = self.ini_parser.get_param(
            'params', 'end_subsentence', param_type='str_array', sep='\\')
        self.settings['params']['end_sentence'] = self.ini_parser.get_param(
            'params', 'end_sentence', param_type='str_array', sep='\\')

    def main(self, args):
        self.set_params(args)
        # print(self.settings)
        text_parser = TextParser(self.errors)

        # input_file_name = self.settings['input']['file_path']
        # enc = self.settings['input']['encoding']
        # params = self.settings['params']
        text_parser.parse_text(self.settings)

        if self.errors.error_occured:
            self.errors.print_errors()
        else:
            print('OK\n')
Ejemplo n.º 2
0
class Template:
    def __init__(self):
        self.errors = Errors()
        self.ini_encoding = 'utf-8'
        self.ini_parser = IniParser(self.errors)
        self.settings = {}

    def read_settings(self, args):
        if len(args) < 2:
            self.errors.raise_error('no ini file path')
        else:
            encoding = self.ini_encoding
            if len(args) > 2:
                encoding = args[2]

            ini_file_path = args[1]
            self.ini_parser.read_ini(args[1], encoding)

    def set_params(self, args):
        tools = Tools(self.errors)
        self.read_settings(args)
        self.settings['g1'] = self.ini_parser.get_param('g1', None)
        self.settings['g2'] = {}
        self.settings['g2']['p1'] = tools.explode(
            ',', self.ini_parser.get_param('g2', 'p1'))
        self.settings['g3'] = {}
        self.settings['g3']['i4'] = self.ini_parser.get_param(
            'g3', 'i4', 'int')

    def main(self, args):
        self.set_params(args)
        print(self.settings)

        if self.errors.error_occured:
            self.errors.print_errors()
        else:
            print('OK\n')
Ejemplo n.º 3
0
class Template:
    def __init__(self):
        self.errors = Errors()
        self.ini_encoding = 'utf-8'
        self.ini_parser = IniParser(self.errors)
        self.settings = {}

    def read_settings(self, args):
        if len(args) < 2:
            self.errors.raise_error('No ini file path')
        else:
            encoding = self.ini_encoding
            if len(args) > 2:
                encoding = args[2]

            ini_file_path = args[1]
            self.ini_parser.read_ini(args[1], encoding)

    def set_params(self, args):
        self.read_settings(args)
        self.settings['g2'] = {}
        self.settings['g3'] = {}
        self.settings['g2']['p1'] = self.ini_parser.get_param(
            'g2', 'p1', 'str_array')
        self.settings['g3'] = self.ini_parser.get_param('g3')

    def main(self, args):
        self.set_params(args)
        print(self.settings)
        g1_p2 = self.ini_parser.get_param('g1', 'p2')
        print(g1_p2)

        if self.errors.error_occured:
            self.errors.print_errors()
        else:
            print('OK\n')
Ejemplo n.º 4
0
class GetFinamData:
    def __init__(self):
        self.errors = Errors()
        self.ini_encoding = 'utf-8'
        self.ini_parser = IniParser(self.errors)
        self.tc_ini_parser = IniParser(self.errors)
        self.settings = {}

    def read_settings(self, args):
        if len(args) < 2:
            self.errors.raise_error('no ini file path')
        else:
            encoding = self.ini_encoding
            if len(args) > 2:
                self.ini_encoding = args[2]

            ini_file_path = args[1]
            self.ini_parser.read_ini(args[1], self.ini_encoding)

    def set_params(self, args):
        tools = Tools(self.errors)
        self.read_settings(args)
        self.settings['common'] = {}
        self.settings['common']['time_frames'] = tools.explode(
            ',', self.ini_parser.get_param('common', 'time_frames'))
        self.settings['common']['output_folder'] = self.ini_parser.get_param(
            'common', 'output_folder')
        self.settings['common'][
            'trading_calendar'] = self.ini_parser.get_param(
                'common', 'trading_calendar')
        self.settings['common'][
            'always_update_past_working_days_number'] = self.ini_parser.get_param(
                'common', 'always_update_past_working_days_number', 'int')

        self.tc_ini_parser.read_ini(
            self.settings['common']['trading_calendar'], self.ini_encoding)
        # self.settings['non_working_days'] = {}
        # self.settings['non_working_days'] = self.tc_ini_parser.get_param('non_working_days')

        self.settings['contracts'] = {}
        tickers = tools.explode(
            ',', self.ini_parser.get_param('contracts', 'tickers'))

        for ticker_idx in range(len(tickers)):
            self.settings['contracts'][ticker_idx] = {}
            self.settings['contracts'][ticker_idx]['ticker'] = tickers[
                ticker_idx]
            self.settings['contracts'][ticker_idx]['list'] = {}
            list = tools.explode(
                ',', self.ini_parser.get_param('contracts',
                                               tickers[ticker_idx]))
            for contract_idx in range(len(list)):
                self.settings['contracts'][ticker_idx]['list'][
                    contract_idx] = self.ini_parser.get_param(
                        list[contract_idx])

    def get_contract(self, _idx=[0, 0]):
        if self.errors.error_occured:
            return None, None, None, None, None, None

        if _idx[0] < len(self.settings['contracts']):
            if _idx[1] < len(self.settings['contracts'][_idx[0]]['list']):
                Ticker = self.settings['contracts'][_idx[0]]['ticker']
                ContractSymbol = self.settings['contracts'][_idx[0]]['list'][
                    _idx[1]]['ContractSymbol']
                ContractTradingSymbol = self.settings['contracts'][
                    _idx[0]]['list'][_idx[1]]['ContractTradingSymbol']
                FirstTradingDay = self.settings['contracts'][_idx[0]]['list'][
                    _idx[1]]['FirstTradingDay']
                LastTradingDay = self.settings['contracts'][_idx[0]]['list'][
                    _idx[1]]['LastTradingDay']
                FinamEm = self.settings['contracts'][_idx[0]]['list'][
                    _idx[1]]['FinamEm']
                _idx[1] += 1
            else:
                _idx[0] += 1
                _idx[1] = 0
                Ticker, ContractSymbol, ContractTradingSymbol, FirstTradingDay, LastTradingDay, FinamEm = self.get_contract(
                    _idx)
        else:
            return None, None, None, None, None, None

        return Ticker, ContractSymbol, ContractTradingSymbol, FirstTradingDay, LastTradingDay, FinamEm

    def shape_finam_url(self, current_trading_day, arch, FinamEm,
                        ContractSymbol, time_frame):
        # url = http://export.finam.ru/SPFB.Eu-3.19_190108_190108.txt?
        # market = 14 # Номер рынка
        # em = 487593 # Номер инструмента
        # code = SPFB.Eu-3.19 # Тикер инструмента
        # apply = 0 #
        # df = 8 # Начальная дата, номер дня (1-31)
        # mf = 0 # Начальная дата, номер месяца (0-11)
        # yf = 2019 # Начальная дата, год
        # from = 08.01.2019 # Начальная дата
        # dt = 8 # Конечная дата, номер дня
        # mt = 0 # Конечная дата, номер месяца
        # yt = 2019 # Конечная дата, год
        # to = 08.01.2019 # Конечная дата
        # p = 3 # Таймфрейм {'tick': 1, 'min': 2, '5min': 3, '10min': 4, '15min': 5, '30min': 6, 'hour': 7, 'daily': 8, 'week': 9, 'month': 10}
        # f = SPFB.Eu-3.19_190108_190108 # Имя сформированного файла
        # e = .txt # Расширение сформированного файла#  возможны варианты — .txt либо .csv
        # cn = SPFB.Eu-3.19 # Имя контракта
        # dtf = 1 # формат даты (1 — ггггммдд, 2 — ггммдд, 3 — ддммгг, 4 — дд/мм/гг, 5 — мм/дд/гг)
        # tmf = 1 # формат времени (1 — ччммсс, 2 — ччмм, 3 — чч: мм: сс, 4 — чч: мм)
        # MSOR = 1 # выдавать время (0 — начала свечи, 1 — окончания свечи)
        # mstime = on # выдавать время (НЕ московское — mstimever=0#  московское — mstime='on', mstimever='1')
        # mstimever = 1 # Коррекция часового пояса
        # sep = 1 # Разделитель полей (1 — запятая (,), 2 — точка (.), 3 — точка с запятой (;), 4 — табуляция (»), 5 — пробел ( ))
        # sep2 = 1 # Разделитель разрядов (1 — нет, 2 — точка (.), 3 — запятая (,), 4 — пробел ( ), 5 — кавычка ('))
        # datf = 1 # Перечень получаемых данных (#1 — TICKER, PER, DATE, TIME, OPEN, HIGH, LOW, CLOSE, VOL#  #2 — TICKER, PER, DATE, TIME, OPEN, HIGH, LOW, CLOSE#  #3 — TICKER, PER, DATE, TIME, CLOSE, VOL#  #4 — TICKER, PER, DATE, TIME, CLOSE#  #5 — DATE, TIME, OPEN, HIGH, LOW, CLOSE, VOL#  #6 — DATE, TIME, LAST, VOL, ID, OPER).
        # at = 1 # добавлять заголовок в файл (0 — нет, 1 — да)

        url = 'http://export.finam.ru/'
        if arch:
            market = '13'
        else:
            market = '14'
        em = FinamEm
        code = 'SPFB.' + ContractSymbol
        apply = '0'
        df = str(current_trading_day.day)
        mf = str(current_trading_day.month - 1)
        yf = str(current_trading_day.year)
        _from = dt.strftime(current_trading_day, '%d.%m.%Y')
        _dt = str(current_trading_day.day)
        mt = str(current_trading_day.month - 1)
        yt = str(current_trading_day.year)
        to = dt.strftime(current_trading_day, '%d.%m.%Y')
        time_frame_codes = {
            '0': '1',
            '1': '2',
            '5': '3',
            '10': '4',
            '15': '5',
            '30': '6',
            '60': '7',
            'D': '8',
            'W': '9',
            'M': '10'
        }
        p = time_frame_codes[time_frame]
        f = code + '_' + dt.strftime(current_trading_day, '%Y-%m-%d')
        e = '.txt'
        cn = code
        dtf = '1'
        tmf = '1'
        MSOR = '1'
        mstime = 'on'
        mstimever = '1'
        sep = '1'
        sep2 = '1'
        datf = '1'
        at = '1'

        url += (f + e + '?market=' + market + '&em=' + em + '&code=' + code +
                '&apply=' + apply + '&df=' + df + '&mf=' + mf + '&yf=' + yf +
                '&from=' + _from + '&dt=' + _dt + '&mt=' + mt + '&yt=' + yt +
                '&to=' + to + '&p=' + p + '&f=' + f + '&e=' + e + '&cn=' + cn +
                '&dtf=' + dtf + '&tmf=' + tmf + '&MSOR=' + MSOR + '&mstime=' +
                mstime + '&mstimever=' + mstimever + '&sep=' + sep + '&sep2=' +
                sep2 + '&datf=' + datf + '&at=' + at + '')

        return url, f + e

    def is_non_working_day(self, current_trading_day):
        ctd_year = str(current_trading_day.year)
        ctd_month = str(current_trading_day.month)
        ctd_day = str(current_trading_day.day)

        tools = Tools(self.errors)
        nw_days = tools.explode(
            ',',
            self.tc_ini_parser.get_param('non_working_days',
                                         ctd_month + '.' + ctd_year))
        if ctd_day in nw_days:
            return True

        return False

    def allow_update(self, now_day, current_trading_day, file_path):
        if self.errors.error_occured:
            return False

        day_cnt = 1
        days_shift = datetime.timedelta(days=0)
        always_update_past_working_days_number = self.settings['common'][
            'always_update_past_working_days_number']
        while always_update_past_working_days_number > 0:
            days_shift = datetime.timedelta(days=day_cnt)
            if not self.is_non_working_day(now_day - days_shift):
                always_update_past_working_days_number -= 1
            day_cnt += 1

        if current_trading_day < now_day - days_shift:
            if not self.is_non_working_day(current_trading_day):
                if not os.path.exists(file_path):
                    return True
        elif current_trading_day <= now_day:
            if not self.is_non_working_day(current_trading_day):
                return True
        return False

    def main(self, args):
        self.set_params(args)

        fs = FileSystem(self.errors)
        now_day = dt.today().date()

        time_frames = self.settings['common']['time_frames']

        while not self.errors.error_occured:
            Ticker, ContractSymbol, ContractTradingSymbol, FirstTradingDay, LastTradingDay, FinamEm = self.get_contract(
            )
            if Ticker:
                first_trading_day = dt.strptime(FirstTradingDay,
                                                '%d.%m.%Y').date()
                last_trading_day = dt.strptime(LastTradingDay,
                                               '%d.%m.%Y').date()
                ltd_year = last_trading_day.year

                # delta = last_trading_day - first_trading_day
                one_day = datetime.timedelta(days=1)
                current_trading_day = first_trading_day
                while not self.errors.error_occured:
                    # for day_cnt in range(delta.days):
                    for time_frame in time_frames:
                        if now_day > last_trading_day:
                            arch = True
                        else:
                            arch = False
                        path = self.settings['common']['output_folder'] + str(
                            ltd_year
                        ) + '/' + Ticker + '/' + ContractSymbol + '/' + time_frame + '/'
                        fs.create_folder_branch(path)
                        url, file = self.shape_finam_url(
                            current_trading_day, arch, FinamEm, ContractSymbol,
                            time_frame)
                        file_path = path + file
                        if self.allow_update(now_day, current_trading_day,
                                             file_path):
                            print(Ticker, ContractSymbol, current_trading_day,
                                  time_frame)
                            try:
                                page = urllib.request.urlopen(url)
                            except Exception as e:
                                self.errors.raise_error('Can\'t open url ' +
                                                        url)
                                break
                            content = page.read()
                            content = content.decode('utf-8').replace('\r', '')

                            try:
                                with open(file_path, "w") as text_file:
                                    print(content, file=text_file)
                            except Exception as e:
                                self.errors.raise_error('Can\'t write file ' +
                                                        file_path)
                                break
                            time.sleep(1)

                    current_trading_day += one_day
                    if current_trading_day > last_trading_day:
                        break
            else:
                break

        # print(self.settings)

        if self.errors.error_occured:
            self.errors.print_errors()
        else:
            print('OK\n')
Ejemplo n.º 5
0
class Plot:
	def __init__(self):
		self.errors = Errors()
		self.ini_encoding = 'utf-8'
		self.ini_parser = IniParser(self.errors)
		self.settings = {}
	
	def read_settings(self, args):
		if len(args) < 2:
			self.errors.raise_error('no ini file path')
		else:
			encoding = self.ini_encoding
			if len(args) > 2:
				encoding = args[2]
				
			ini_file_path = args[1]
			self.ini_parser.read_ini(args[1], encoding)
	
	def set_params(self, args):
		self.read_settings(args)
		tools = Tools(self.errors)
		
		self.settings['input'] = {}
		self.settings['input']['file_path'] = self.ini_parser.get_param('input', 'file_path')
		self.settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
		
		self.settings['output'] = {}
		self.settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
		
		input_feed_format = self.settings['input']['input_feed_format']
		self.settings[input_feed_format] = {}
		self.settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
		self.settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
		self.settings[input_feed_format]['columns'] = tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
		self.settings[input_feed_format]['column_separator'] = tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
		self.settings[input_feed_format]['column_data_types'] = tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
		
		self.settings['plotter'] = {}
		self.settings['plotter']['fig_folder'] = self.settings['output']['folder'] + self.ini_parser.get_param('plotter', 'fig_folder')
		self.settings['plotter']['x_ticks_data_columns'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks_data_columns'))
		self.settings['plotter']['x_ticks'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks'))
		self.settings['plotter']['x_labels_format'] = tools.explode(',', self.ini_parser.get_param('plotter', 'x_labels_format'))
		self.settings['plotter']['series'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series'))
		self.settings['plotter']['subplot_height_share'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'subplot_height_share')))
		self.settings['plotter']['subplots_number'] = self.ini_parser.get_param('plotter', 'subplots_number', 'int') 
		self.settings['plotter']['seria_to_subbplot_binding'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'seria_to_subbplot_binding')))
		
		def clone_value(value, number):
			return number * [value]
		
		ignoring_missed_data_line = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'ignoring_missed_data_line'), len(self.settings['plotter']['series'])))
		series_linewidth = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_linewidth'), len(self.settings['plotter']['series'])))
		series_markersize = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_markersize'), len(self.settings['plotter']['series'])))
		series_marker = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_marker'), len(self.settings['plotter']['series'])))
		series_color = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_color'), len(self.settings['plotter']['series'])))
		series_alpha = tools.implode(',', clone_value(self.ini_parser.get_param('plotter_default', 'series_alpha'), len(self.settings['plotter']['series'])))
		
		self.settings['plotter']['ignoring_missed_data_line'] = tools.bool_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'ignoring_missed_data_line', error_ignoring = True, default_param_value = ignoring_missed_data_line)))
		self.settings['plotter']['series_linewidth'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_linewidth', error_ignoring = True, default_param_value = series_linewidth)))
		self.settings['plotter']['series_marker'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series_marker', error_ignoring = True, default_param_value = series_marker))
		self.settings['plotter']['series_markersize'] = tools.int_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_markersize', error_ignoring = True, default_param_value = series_markersize)))
		self.settings['plotter']['series_color'] = tools.explode(',', self.ini_parser.get_param('plotter', 'series_color', error_ignoring = True, default_param_value = series_color))
		self.settings['plotter']['series_alpha'] = tools.float_arr(tools.explode(',', self.ini_parser.get_param('plotter', 'series_alpha', error_ignoring = True, default_param_value = series_alpha)))
		
		#check several params
		def check_elements_number(section1, param1, section2, param2):
			if self.errors.error_occured:
				return None
			elements_number1 = len(self.settings[section1][param1])
			elements_number2 = self.settings[section2][param2]
			
			if elements_number1 != elements_number2:
				self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] value(' +  str(elements_number2) + ')')

		check_elements_number('plotter', 'subplot_height_share', 'plotter', 'subplots_number')
		
		def compare_elements_number(section1, param1, section2, param2):
			if self.errors.error_occured:
				return None
			elements_number1 = len(self.settings[section1][param1])
			elements_number2 = len(self.settings[section2][param2])
			if elements_number1 != elements_number2:
				self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] elements number(' +  str(elements_number2) + ')')

		compare_elements_number(input_feed_format, 'columns', input_feed_format, 'column_data_types')
		compare_elements_number('plotter', 'x_ticks_data_columns', 'plotter', 'x_ticks')
		compare_elements_number('plotter', 'x_ticks_data_columns', 'plotter', 'x_labels_format')
		compare_elements_number('plotter', 'series', 'plotter', 'seria_to_subbplot_binding')
		
	def main(self, args):
		self.set_params(args)
		data_stream = DataStream(self.errors)
		input_file_path = self.settings['input']['file_path']
		input_feed_format = self.settings[self.settings['input']['input_feed_format']]
		data_stream.open_stream(input_file_path, input_feed_format)
		data = data_stream.read_all(input_feed_format)
		data_stream.close_stream()
		
		fig_name = '0000'
		plotter = Plotter(self.errors)
		plotter.plot_series(data, self.settings, fig_name)
		
		# print(data)
		
		if self.errors.error_occured:
			self.errors.print_errors()
		else:
			print('OK\n')
Ejemplo n.º 6
0
class SettingsReader:
	def __init__(self, errors):
		self.errors = errors
		self.settings = {}
		self.ini_encoding = 'utf-8'
		self.ini_parser = IniParser(self.errors)
		self.tools = Tools(self.errors)
		
	def check_elements_number(self, settings, section1, param1, section2, param2):
		elements_number1 = len(settings[section1][param1])
		elements_number2 = settings[section2][param2]
		
		if elements_number1 != elements_number2:
			self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] value(' +  str(elements_number2) + ')')
	
	def compare_elements_number(self, settings, section1, param1, section2, param2):
		elements_number1 = len(settings[section1][param1])
		elements_number2 = len(settings[section2][param2])
		if elements_number1 != elements_number2:
			self.errors.raise_error('[' + section1 + '][' + param1 + '] elements number(' + str(elements_number1) + ') is not equal [' + section2 + '][' + param2 + '] elements number(' +  str(elements_number2) + ')')
		
	def clone_value(self, value, number):
		return number * [value]
	
	def read_ArbitrageSettings(self, settings, ini_file_path, encoding):
		if self.errors.error_occured:
			return None
		
		self.ini_parser.read_ini(ini_file_path, encoding)
		settings['input'] = {}
		settings['input']['file_path'] = self.ini_parser.get_param('input', 'file_path')
		settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
		settings['input']['moex_currency_file'] = self.ini_parser.get_param('input', 'moex_currency_file')
		settings['input']['moex_currency_feed_format'] = self.ini_parser.get_param('input', 'moex_currency_feed_format')

		settings['output'] = {}
		settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
		settings['output']['file'] = self.ini_parser.get_param('output', 'file')
		settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
		
		input_feed_format = settings['input']['input_feed_format']
		settings[input_feed_format] = {}
		settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
		settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
		settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
		settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
		settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
		
		moex_currency_feed_format = settings['input']['moex_currency_feed_format']
		settings[moex_currency_feed_format] = {}
		settings[moex_currency_feed_format]['encoding'] = self.ini_parser.get_param(moex_currency_feed_format, 'encoding')
		settings[moex_currency_feed_format]['header_lines_number'] = self.ini_parser.get_param(moex_currency_feed_format, 'header_lines_number', 'int')
		settings[moex_currency_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(moex_currency_feed_format, 'columns'))
		settings[moex_currency_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(moex_currency_feed_format, 'column_separator'))
		settings[moex_currency_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(moex_currency_feed_format, 'column_data_types'))
		
		output_feed_format = settings['output']['output_feed_format']
		settings[output_feed_format] = {}
		settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
		settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
		settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
		settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
		settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
		
		#check several params
		self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
		self.compare_elements_number(settings, moex_currency_feed_format, 'columns', moex_currency_feed_format, 'column_data_types')
		self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
		
	def read_JoinData_settings(self, settings, ini_file_path, encoding):
		if self.errors.error_occured:
			return None
		
		self.ini_parser.read_ini(ini_file_path, encoding)
		settings['input'] = {}
		settings['input']['folder'] = self.ini_parser.get_param('input', 'folder')
		settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
		settings['input']['columns'] = self.tools.explode(',', self.ini_parser.get_param('input', 'columns'))
		settings['input']['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param('input', 'column_data_types'))
		
		settings['input']['date_col'] = self.ini_parser.get_param('input', 'date_col')
		settings['input']['start_time'] = self.ini_parser.get_param('input', 'start_time')
		settings['input']['stop_time'] = self.ini_parser.get_param('input', 'stop_time')
		settings['input']['step_time'] = self.ini_parser.get_param('input', 'step_time')
		settings['input']['exclude_time'] = self.tools.explode(',', self.ini_parser.get_param('input', 'exclude_time'))
		
		settings['output'] = {}
		settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
		settings['output']['file'] = self.ini_parser.get_param('output', 'file')
		settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
		
		input_feed_format = settings['input']['input_feed_format']
		settings[input_feed_format] = {}
		settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
		settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
		settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
		settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
		settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
		
		output_feed_format = settings['output']['output_feed_format']
		settings[output_feed_format] = {}
		settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
		settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
		# settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
		settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
		# settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
		
		#check several params
		self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
		# self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
		self.compare_elements_number(settings, 'input', 'columns', 'input', 'column_data_types')
		
	
	def read_ConcatinateData_settings(self, settings, ini_file_path, encoding):
		if self.errors.error_occured:
			return None
		
		self.ini_parser.read_ini(ini_file_path, encoding)
		settings['input'] = {}
		settings['input']['folder'] = self.ini_parser.get_param('input', 'folder')
		settings['input']['input_feed_format'] = self.ini_parser.get_param('input', 'input_feed_format')
		
		settings['output'] = {}
		settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
		settings['output']['file'] = self.ini_parser.get_param('output', 'file')
		settings['output']['output_feed_format'] = self.ini_parser.get_param('output', 'output_feed_format')
		
		input_feed_format = settings['input']['input_feed_format']
		settings[input_feed_format] = {}
		settings[input_feed_format]['encoding'] = self.ini_parser.get_param(input_feed_format, 'encoding')
		settings[input_feed_format]['header_lines_number'] = self.ini_parser.get_param(input_feed_format, 'header_lines_number', 'int')
		settings[input_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'columns'))
		settings[input_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(input_feed_format, 'column_separator'))
		settings[input_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(input_feed_format, 'column_data_types'))
		
		output_feed_format = settings['output']['output_feed_format']
		settings[output_feed_format] = {}
		settings[output_feed_format]['encoding'] = self.ini_parser.get_param(output_feed_format, 'encoding')
		settings[output_feed_format]['header_lines_number'] = self.ini_parser.get_param(output_feed_format, 'header_lines_number', 'int')
		settings[output_feed_format]['columns'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'columns'))
		settings[output_feed_format]['column_separator'] = self.tools.escape_sequence(self.ini_parser.get_param(output_feed_format, 'column_separator'))
		settings[output_feed_format]['column_data_types'] = self.tools.explode(',', self.ini_parser.get_param(output_feed_format, 'column_data_types'))
		
		#check several params
		self.compare_elements_number(settings, input_feed_format, 'columns', input_feed_format, 'column_data_types')
		self.compare_elements_number(settings, output_feed_format, 'columns', output_feed_format, 'column_data_types')
		
	def read_PlotterSettings(self, settings, ini_file_path, encoding):
		if self.errors.error_occured:
			return None
			
		settings['output'] = {}
		settings['output']['folder'] = self.ini_parser.get_param('output', 'folder')
			
		settings['plotter'] = {}
		settings['plotter']['fig_folder'] = settings['output']['folder'] + self.ini_parser.get_param('plotter', 'fig_folder')
		settings['plotter']['x_ticks_data_columns'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks_data_columns'))
		settings['plotter']['x_ticks'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_ticks'))
		settings['plotter']['x_labels_format'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'x_labels_format'))
		settings['plotter']['series'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series'))
		settings['plotter']['subplot_height_share'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'subplot_height_share')))
		settings['plotter']['subplots_number'] = self.ini_parser.get_param('plotter', 'subplots_number', 'int') 
		settings['plotter']['seria_to_subbplot_binding'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'seria_to_subbplot_binding')))
		
		ignoring_missed_data_line = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'ignoring_missed_data_line'), len(settings['plotter']['series'])))
		series_linewidth = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_linewidth'), len(settings['plotter']['series'])))
		series_markersize = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_markersize'), len(settings['plotter']['series'])))
		series_marker = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_marker'), len(settings['plotter']['series'])))
		series_color = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_color'), len(settings['plotter']['series'])))
		series_alpha = self.tools.implode(',', self.clone_value(self.ini_parser.get_param('plotter_default', 'series_alpha'), len(settings['plotter']['series'])))
		
		settings['plotter']['ignoring_missed_data_line'] = self.tools.bool_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'ignoring_missed_data_line', error_ignoring = True, default_param_value = ignoring_missed_data_line)))
		settings['plotter']['series_linewidth'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_linewidth', error_ignoring = True, default_param_value = series_linewidth)))
		settings['plotter']['series_marker'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_marker', error_ignoring = True, default_param_value = series_marker))
		settings['plotter']['series_markersize'] = self.tools.int_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_markersize', error_ignoring = True, default_param_value = series_markersize)))
		settings['plotter']['series_color'] = self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_color', error_ignoring = True, default_param_value = series_color))
		settings['plotter']['series_alpha'] = self.tools.float_arr(self.tools.explode(',', self.ini_parser.get_param('plotter', 'series_alpha', error_ignoring = True, default_param_value = series_alpha)))
		
		#check several params
		self.check_elements_number(settings, 'plotter', 'subplot_height_share', 'plotter', 'subplots_number')
		self.compare_elements_number(settings, 'plotter', 'x_ticks_data_columns', 'plotter', 'x_ticks')
		self.compare_elements_number(settings, 'plotter', 'x_ticks_data_columns', 'plotter', 'x_labels_format')
		self.compare_elements_number(settings, 'plotter', 'series', 'plotter', 'seria_to_subbplot_binding')
Ejemplo n.º 7
0
class GetMoexData:
    def __init__(self):
        self.errors = Errors()
        self.ini_encoding = 'utf-8'
        self.ini_parser = IniParser(self.errors)
        self.tc_ini_parser = IniParser(self.errors)
        self.settings = {}

    def read_settings(self, args):
        if len(args) < 2:
            self.errors.raise_error('no ini file path')
        else:
            encoding = self.ini_encoding
            if len(args) > 2:
                self.ini_encoding = args[2]

            ini_file_path = args[1]
            self.ini_parser.read_ini(args[1], self.ini_encoding)

    def set_params(self, args):
        self.read_settings(args)
        self.settings['output'] = {}
        self.settings['output']['file_path'] = self.ini_parser.get_param(
            'output', 'file_path')

        self.settings['period'] = {}
        self.settings['period']['start_date'] = self.ini_parser.get_param(
            'period', 'start_date')

    def shape_moex_url(self, start_date, end_date):
        url = 'https://www.moex.com/export/derivatives/currency-rate.aspx?'
        url = (url + 'language=en' + '&currency=USD/RUB' + '&moment_start=' +
               start_date + '&moment_end=' + end_date)
        # print(urllib.parse.quote(url))
        return url  #urllib.parse.quote(url)

    def extract_rates(self, content):
        saved_content = '<CURRENCY>,<DATE>,<TIME>,<RATE>\n'
        rates = []
        while True:
            rate_moment_pos = content.find('rate moment')

            value_pos = content.find('value')
            end_str_pos = content.find('/>')

            rate_moment = content[rate_moment_pos + 13:value_pos - 2]
            value = content[value_pos + 7:end_str_pos - 2]
            if rate_moment_pos == -1:
                break
            else:
                moment_date = rate_moment.split(' ')[0]
                monent_time = rate_moment.split(' ')[1]

                moment_date = dt.strftime(dt.strptime(moment_date, '%Y-%m-%d'),
                                          '%Y%m%d')
                monent_time = dt.strftime(dt.strptime(monent_time, '%H:%M:%S'),
                                          '%H%M%S')
                rates.append('USDRUB' + ',' + moment_date + ',' + monent_time +
                             ',' + value)

            content = content[end_str_pos + 2:]

        rates.sort()
        saved_content += '\n'.join(rates)

        return saved_content

    def main(self, args):
        self.set_params(args)

        fs = FileSystem(self.errors)

        file_path = self.settings['output']['file_path']
        start_date = self.settings['period']['start_date']

        if not self.errors.error_occured:
            start_date = dt.strftime(dt.strptime(start_date, '%d.%m.%Y'),
                                     '%Y-%m-%d')
            end_date = dt.strftime((dt.today()), '%Y-%m-%d')
            url = self.shape_moex_url(start_date, end_date)
            content = ''
            try:
                page = urllib.request.urlopen(url)
                content = page.read()
                content = content.decode('utf-8').replace('\r', '')
            except Exception as e:
                self.errors.raise_error('Can\'t open url ' + url + '\n' +
                                        str(e))

            content = self.extract_rates(content)
            folder_path, file_name = fs.split_file_path(file_path)

            fs.create_folder_branch(folder_path)
            try:
                with open(file_path, "w") as text_file:
                    print(content, file=text_file)
            except Exception as e:
                self.errors.raise_error('Can\'t write file ' + file_path)

        if self.errors.error_occured:
            self.errors.print_errors()
        else:
            print('OK\n')