def parse(self, response): companies = dict() companies_info = response.css('table#cr1 > tbody').xpath('./tr[contains(@id,"pair")]') for company in companies_info: name_el = company.xpath('./td[2]/a') name = name_el.xpath('./text()').extract_first() full_name = name_el.xpath('./@title').extract_first() companies[name] = dict() current_price = company.xpath('./td[contains(@class,"last")]/text()').extract_first() current_price = Formatter.format_price(current_price) high_price = company.xpath('./td[contains(@class,"high")]/text()').extract_first() high_price = Formatter.format_price(high_price) low_price = company.xpath('./td[contains(@class,"low")]/text()').extract_first() low_price = Formatter.format_price(low_price) volume = company.xpath('./td[contains(@class,"turnover")]/text()').extract_first() volume = Formatter.format_volume(volume) update_time = int(company.xpath('./td[contains(@class,"time")]/@data-value').extract_first()) companies[name]["current_price"] = current_price companies[name]["high_price"] = high_price companies[name]["low_price"] = low_price companies[name]["volume"] = volume companies[name]["full_name"] = full_name companies[name]["time"] = update_time db = Database() db.insert_companies(companies)
def get_formatted_dataset(self, name, file_limit, type, anim_type): formatter = Formatter(anim_type=anim_type) set_x, set_y = formatter.get_dataset(type=type, name=name, file_limit=file_limit) for index, dict in enumerate(set_y): set_y[index] = self.format_y(dict) if self.flatten_input: for index, dict in enumerate(set_x): set_x[index] = dict.flatten() return set_x, set_y
def get_value(path): path, _format = Formatter.get_format(path) head, tail = split(path) if head in handlers: value = handlers[head].handle(tail) if _format is not None: value = Formatter.format(_format, value) return value else: raise Exception("Element '" + head + "' in '" + path + "' is not supported")
def scrape(self): """Main programm.""" formatter = Formatter(self._language) for index, (main_title, redirects) in enumerate(self._titles.items()): # ensures to continue where script left off if self._start_index > index: continue self._notify(index, main_title) content = formatter.format_with_title(main_title, redirects, pretty_print=False) if not content: # there are some files that need to be skipped continue self._save_content(main_title, content) self._save_state(main_title) print("Sucessfully scrapped.")
import sql_parser from format import Formatter parser = sql_parser.BigQueryViewParser() sql = """ with my_cte as (select sum(case when a=1 then 1 else 0 end) as pivoted from table) select * from my_cte """ ast = parser._parse(sql) f = Formatter() f.format(ast) f.document.pprint()