def parse(self, response):
companies = dict()
companies_info = response.css('table#cr1 > tbody').xpath('./tr[contains(@id,"pair")]')
for company in companies_info:
name_el = company.xpath('./td[2]/a')
name = name_el.xpath('./text()').extract_first()
full_name = name_el.xpath('./@title').extract_first()
companies[name] = dict()
current_price = company.xpath('./td[contains(@class,"last")]/text()').extract_first()
current_price = Formatter.format_price(current_price)
high_price = company.xpath('./td[contains(@class,"high")]/text()').extract_first()
high_price = Formatter.format_price(high_price)
low_price = company.xpath('./td[contains(@class,"low")]/text()').extract_first()
low_price = Formatter.format_price(low_price)
volume = company.xpath('./td[contains(@class,"turnover")]/text()').extract_first()
volume = Formatter.format_volume(volume)
update_time = int(company.xpath('./td[contains(@class,"time")]/@data-value').extract_first())
companies[name]["current_price"] = current_price
companies[name]["high_price"] = high_price
companies[name]["low_price"] = low_price
companies[name]["volume"] = volume
companies[name]["full_name"] = full_name
companies[name]["time"] = update_time
db = Database()
db.insert_companies(companies)
def get_formatted_dataset(self, name, file_limit, type, anim_type):
formatter = Formatter(anim_type=anim_type)
set_x, set_y = formatter.get_dataset(type=type, name=name, file_limit=file_limit)
for index, dict in enumerate(set_y):
set_y[index] = self.format_y(dict)
if self.flatten_input:
for index, dict in enumerate(set_x):
set_x[index] = dict.flatten()
return set_x, set_y
def get_value(path):
path, _format = Formatter.get_format(path)
head, tail = split(path)
if head in handlers:
value = handlers[head].handle(tail)
if _format is not None:
value = Formatter.format(_format, value)
return value
else:
raise Exception("Element '" + head + "' in '" + path + "' is not supported")
def get_value(path):
path, _format = Formatter.get_format(path)
head, tail = split(path)
if head in handlers:
value = handlers[head].handle(tail)
if _format is not None:
value = Formatter.format(_format, value)
return value
else:
raise Exception("Element '" + head + "' in '" + path + "' is not supported")
def get_formatted_dataset(self, name, file_limit, type, anim_type):
formatter = Formatter(anim_type=anim_type)
set_x, set_y = formatter.get_dataset(type=type,
name=name,
file_limit=file_limit)
for index, dict in enumerate(set_y):
set_y[index] = self.format_y(dict)
if self.flatten_input:
for index, dict in enumerate(set_x):
set_x[index] = dict.flatten()
return set_x, set_y
def scrape(self):
"""Main programm."""
formatter = Formatter(self._language)
for index, (main_title, redirects) in enumerate(self._titles.items()):
# ensures to continue where script left off
if self._start_index > index:
continue
self._notify(index, main_title)
content = formatter.format_with_title(main_title, redirects, pretty_print=False)
if not content: # there are some files that need to be skipped
continue
self._save_content(main_title, content)
self._save_state(main_title)
print("Sucessfully scrapped.")
import sql_parser from format import Formatter parser = sql_parser.BigQueryViewParser() sql = """ with my_cte as (select sum(case when a=1 then 1 else 0 end) as pivoted from table) select * from my_cte """ ast = parser._parse(sql) f = Formatter() f.format(ast) f.document.pprint()