def get_weather_data_from_city_page_source(self, url, page_source):
print(hue.blue(f'Scrapying Climatempo page source at {url}'))
page = BeautifulSoup(page_source, 'html.parser')
table_attrs = {
'class': 'left top20 small-12 border-none',
}
table = page.find('table', attrs=table_attrs)
metric_trs = table.find_all('tr')
last_month_measured_weather_tr = metric_trs[-1]
last_mesured_month_metric_tds = last_month_measured_weather_tr\
.find_all('td')
city_name_p = page.find('p', attrs={'data-reveal-id': 'geolocation'})
city_name_span = city_name_p.find_all('span')[1]
city_name, city_state = city_name_span.text.split('-')
clean_temp = lambda temp: re.sub(r'\W+', '', temp)
data = {
'city': city_name.strip(),
'state': city_state.strip(),
'month': last_mesured_month_metric_tds[0].text,
'min_temp': clean_temp(last_mesured_month_metric_tds[1].text),
'max_temp': clean_temp(last_mesured_month_metric_tds[2].text),
'rain': last_mesured_month_metric_tds[3].text,
}
print(
hue.green(
f'Scraped Climatempo page source at {url} - Data: {data}'))
return data
def _optimize_file(path):
tinify.key = get_api_key()
with yaspin(text="Optimizing", color="cyan") as spinner:
try:
before_size = getsize(path)
source = tinify.from_file(path)
source.to_file(path)
after_size = getsize(path)
# compare
if before_size == after_size:
before = before_size
after = after_size
else:
before = huepy.blue(before_size)
after = huepy.yellow(after_size)
log_msg = "{0} {1} → {2}"
msg = log_msg.format(os.path.basename(path), before, after)
log = logger.info(msg, flag=SUCCESS)
except (ClientError, ServerError, ConnectionError):
log = logger.error(os.path.basename(path), flag=FAILURE)
finally:
spinner.write(log)
def scrape(self, url):
print(hue.blue(f'Running ClimaTempoMetricScraper for url at {url}'))
driver = self.get_driver()
climatempo_city_page = self.get_city_page_source(driver, url)
climatempo_city_data = self.get_weather_data_from_city_page_source(
url, climatempo_city_page)
print(hue.orange(f'Closing driver for Climatempo url at {url}'))
driver.quit()
metric = ClimaTempoMetric(**climatempo_city_data)
return metric
def scrape(self, url):
print(
hue.blue(
f'Runnning GoogleClimaTempoCityLinkScraper for url at {url}'))
driver = self.get_driver()
google_page_source = self.get_page_source(driver, url)
climatempo_city_link = self.get_climatempo_city_link(
url, google_page_source)
print(hue.orange(f'Closing driver for Google url at {url}'))
driver.quit()
return climatempo_city_link
def get_climatempo_city_link(self, url, page_source):
print(hue.blue(f'Scraping Google page at {url}'))
page = BeautifulSoup(page_source, 'html.parser')
css_selector = 'a[href*=/url?q=https://www.climatempo.com.br/climatologia/]'
climatempo_link_tag = page.select_one(css_selector)
if climatempo_link_tag is None:
print(hue.yellow(f'Climatempo link not found on Google at {url}'))
return ''
climatempo_link = self.get_climatempo_link_from_tag(
climatempo_link_tag)
print(
hue.green(
f'Climatempo link {climatempo_link} scraped on google at {url}'
))
return climatempo_link
def making_dir(path, to_dir):
run(' Making {} directory'.format(huepy.blue(join(path,
to_dir))).replace(
'\\', '/'))
def auto_log(name, from_dir, to_dir):
run('{} from {} to {}'.format(huepy.green(name), huepy.blue(from_dir),
huepy.blue(to_dir)).replace('\\', '/'))
def get_page_source(self, driver, url):
print(hue.blue(f'Fetching Google page at {url}'))
driver.get(url)
return driver.page_source
def get_city_page_source(self, driver, url):
print(hue.blue(f'Fetching Climatempo at {url}'))
driver.get(url)
return driver.page_source
def controlActsMsg():
print(bold(blue("#----- Controls ------ #")))
print("1. Up Arrow key -> Accelerate")
print("2. Down Arrow Key -> Decelerate")
print("3. Left or Right Arrow Key -> Lane Change")
print(bold(blue("#----- Controls ------ #")))
#!/usr/bin/env python
# coding: utf-8
import huepy
SUCCESS = huepy.blue("SUCCESS")
FAILURE = huepy.red("FAILURE")
class Logger(object):
def info(self, msg, flag=None):
if flag:
return huepy.good("{0} {1}".format(flag, msg))
else:
return huepy.good(msg)
def warn(self, msg):
return huepy.info(msg)
def error(self, msg, flag=None):
if flag:
return huepy.bad("{0} {1}".format(flag, msg))
else:
return huepy.bad(msg)
