def get_individual_atlantsolia_prices(): relation = globs.ATLANTSOLIA_LOCATION_RELATION url = 'https://www.atlantsolia.is/stodvaverd/' res = requests.get(url, headers=utils.headers()) html_text = res.content html = lxml.etree.fromstring(html_text, lxml.etree.HTMLParser()) div_prices = html.find( './/*[@id="content"]/div/div/div/div[2]/div/div/table/tbody') prices = {} for div_price in div_prices: key = relation[div_price[0][0].text] bensin95 = float(div_price[1][0].text.replace(',', '.')) diesel = float(div_price[2][0].text.replace(',', '.')) bensin95_discount = int( (bensin95 - globs.ATLANTSOLIA_MINIMUM_DISCOUNT) * 10) / 10.0 diesel_discount = int( (diesel - globs.ATLANTSOLIA_MINIMUM_DISCOUNT) * 10) / 10.0 if key in globs.ATLANTSOLIA_DISCOUNTLESS_STATIONS: bensin95_discount = None diesel_discount = None prices[key] = { 'bensin95': bensin95, 'diesel': diesel, 'bensin95_discount': bensin95_discount, 'diesel_discount': diesel_discount } return prices
def get_individual_olis_prices(): url = 'https://www.olis.is/solustadir/thjonustustodvar/eldsneytisverd/' res = requests.get(url, headers=utils.headers()) html = lxml.etree.fromstring(res.content.decode('utf-8'), lxml.etree.HTMLParser()) data = {'stations': {}, 'highest': {'bensin95': None, 'diesel': None}} price_table = html.find( './/table') # theres just one table element, let's use that ofc for row in price_table.findall('.//tr'): if len(row.findall('.//td')) < 3: continue if row.findall('.//td')[0].text.strip() == '': continue name = row.findall('.//td')[0].text.strip() station_key = globs.OLIS_LOCATION_RELATION[name] bensin = None if row.findall('.//td')[1].text.strip() != '': bensin = float( row.findall('.//td')[1].text.strip().replace(',', '.')) diesel = None if row.findall('.//td')[2].text.strip() != '': diesel = float( row.findall('.//td')[2].text.strip().replace(',', '.')) data['stations'][station_key] = {'bensin95': bensin, 'diesel': diesel} if data['highest']['bensin95'] is None or data['highest'][ 'bensin95'] < bensin: data['highest']['bensin95'] = bensin if data['highest']['diesel'] is None or data['highest'][ 'diesel'] < diesel: data['highest']['diesel'] = diesel assert (data['highest']['bensin95'] is not None) assert (data['highest']['diesel'] is not None) for name in data['stations']: # fallback to highest provided price if for some reason it's not provided ._. if data['stations'][name]['bensin95'] is None: data['stations'][name]['bensin95'] = data['highest']['bensin95'] if data['stations'][name]['diesel'] is None: data['stations'][name]['diesel'] = data['highest']['diesel'] prices = {} olis_stations = utils.load_json( os.path.join(os.path.dirname(os.path.realpath(__file__)), '../stations/olis.json')) for key in olis_stations: if key in data['stations']: bensin95 = data['stations'][key]['bensin95'] diesel = data['stations'][key]['diesel'] else: bensin95 = data['highest']['bensin95'] diesel = data['highest']['diesel'] bensin95_discount = int( (bensin95 - globs.OLIS_MINIMUM_DISCOUNT) * 10) / 10.0 diesel_discount = int( (diesel - globs.OLIS_MINIMUM_DISCOUNT) * 10) / 10.0 prices[key] = { 'bensin95': bensin95, 'diesel': diesel, 'bensin95_discount': bensin95_discount, 'diesel_discount': diesel_discount } return prices
def get_individual_daelan_prices(): headers = utils.headers() session = requests.Session() res = session.get('https://daelan.is/', headers=headers) html = lxml.etree.fromstring(res.content.decode('utf-8'), lxml.etree.HTMLParser()) price_info_container = html.find('.//div[@id="gas-price-info-container"]') bensin95 = None diesel = None prices = {} for column in price_info_container.findall('.//tr'): if len(column.getchildren()) != 3: continue if column[0].text == 'Bensínstöð': continue # skip header station_name = column[0].text bensin_txt = column[1].text diesel_txt = column[2].text bensin95 = float(bensin_txt.replace(' kr.', '').replace(',', '.')) diesel = float(diesel_txt.replace(' kr.', '').replace(',', '.')) key = globs.DAELAN_LOCATION_RELATION[station_name] prices[key] = { 'bensin95': bensin95, 'diesel': diesel, 'bensin95_discount': None, 'diesel_discount': None } return prices
def get_individual_orkan_prices(): url = 'https://www.orkan.is/orkan/orkustodvar/' res = requests.get(url, headers=utils.headers()) html = lxml.etree.fromstring(res.content.decode('utf-8'), lxml.etree.HTMLParser()) div_element = html.find('.//div[@class="accordion__container"]') territories = div_element.findall('.//div[@class="accordion__child"]') prices = {} key = None for territory in territories: for station in territory: station_name = station[0][0].text.strip() bensin95 = float(station[1].text.replace(',', '.')) diesel = float(station[2].text.replace(',', '.')) key = globs.ORKAN_LOCATION_RELATION[station_name] # Orkan has a 3-step discount system controlled by your # spendings on gas from them the month before # See more info here: https://www.orkan.is/Afslattarthrep # For consistency we just use the minimum default discount bensin95_discount = bensin95 - globs.ORKAN_MINIMUM_DISCOUNT diesel_discount = diesel - globs.ORKAN_MINIMUM_DISCOUNT if key in globs.ORKAN_DISCOUNTLESS_STATIONS: bensin95_discount = None diesel_discount = None prices[key] = { 'bensin95': bensin95, 'diesel': diesel, 'bensin95_discount': bensin95_discount, 'diesel_discount': diesel_discount } return prices
def get_individual_ob_prices(): url = 'https://www.ob.is/eldsneytisverd/' res = requests.get(url, headers=utils.headers()) html = lxml.etree.fromstring(res.content.decode('utf-8'), lxml.etree.HTMLParser()) data = {'stations': {}, 'highest': {'bensin95': None, 'diesel': None}} price_table = html.find('.//table[@id="gas-prices"]') for row in price_table.findall('.//tr'): if len(row.findall('.//td')) == 0: continue if row.findall('.//td')[0].get('style') == 'border:0px;': continue name = row.findall('.//td')[0].text.strip() if name == 'Ketilás í Fljótum': continue # throw this one for now, only diesel, needs investigation station_key = globs.OB_LOCATION_RELATION[name] bensin = float(row.findall('.//td')[1].text.strip().replace(',', '.')) diesel = float(row.findall('.//td')[2].text.strip().replace(',', '.')) data['stations'][station_key] = {'bensin95': bensin, 'diesel': diesel} if data['highest']['bensin95'] is None or data['highest'][ 'bensin95'] < bensin: data['highest']['bensin95'] = bensin if data['highest']['diesel'] is None or data['highest'][ 'diesel'] < diesel: data['highest']['diesel'] = diesel prices = {} ob_stations = utils.load_json( os.path.join(os.path.dirname(os.path.realpath(__file__)), '../stations/ob.json')) now = datetime.datetime.now() end = datetime.datetime.strptime(globs.OB_EXTRA_DISCOUNT_UNTIL, '%Y-%m-%dT%H:%M') for key in ob_stations: if key in data['stations']: bensin95 = data['stations'][key]['bensin95'] diesel = data['stations'][key]['diesel'] else: bensin95 = data['highest']['bensin95'] diesel = data['highest']['diesel'] bensin95_discount = int( (bensin95 - globs.OB_MINIMUM_DISCOUNT) * 10) / 10.0 diesel_discount = int((diesel - globs.OB_MINIMUM_DISCOUNT) * 10) / 10.0 if key in globs.OB_DISCOUNTLESS_STATIONS: bensin95_discount = None diesel_discount = None if key in globs.OB_EXTRA_DISCOUNT_STATIONS and now < end: if bensin95_discount is not None: bensin95_discount = int( (bensin95 - globs.OB_EXTRA_DISCOUNT_AMOUNT) * 10) / 10.0 if diesel_discount is not None: diesel_discount = int( (diesel - globs.OB_EXTRA_DISCOUNT_AMOUNT) * 10) / 10.0 prices[key] = { 'bensin95': bensin95, 'diesel': diesel, 'bensin95_discount': bensin95_discount, 'diesel_discount': diesel_discount } return prices
def get_individual_n1_prices(): url_eldsneyti = 'https://www.n1.is/thjonusta/eldsneyti/daeluverd/' url_eldsneyti_api = 'https://www.n1.is/umbraco/api/fuel/getfuelprices' headers = utils.headers() session = requests.Session() session.get(url_eldsneyti, headers=headers) headers_eldsneyti_api = { 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.9,is-IS;q=0.8,is;q=0.7', 'Connection': 'keep-alive', 'Content-Length': '0', 'Host': 'www.n1.is', 'Origin': 'https://www.n1.is', 'Referer': 'https://www.n1.is/thjonusta/eldsneyti/daeluverd/', 'User-Agent': headers['User-Agent'] } res = session.post(url_eldsneyti_api, data='', headers=headers_eldsneyti_api) res.raise_for_status() stations = res.json() prices = {} names_ignore_words = [ 'Þjónustustöð', 'Sjálfsafgreiðslustöð', 'Sjálfsafgreiðsla' ] for station in stations: # <get-name> name = station[ 'Name'] # sometimes a bit dirty, so we attempt to clean it up for word in names_ignore_words: name = name.replace(word, '') name = name.replace('-', ' ') name = ' '.join(name.split()) # </get-name> key = globs.N1_LOCATION_RELATION[name] bensin95 = float(station['GasPrice'].replace(',', '.')) bensin95_discount = bensin95 - globs.N1_DISCOUNT diesel = float(station['DiselPrice'].replace(',', '.')) diesel_discount = diesel - globs.N1_DISCOUNT prices[key] = { 'bensin95': bensin95, 'diesel': diesel, 'bensin95_discount': bensin95_discount, 'diesel_discount': diesel_discount } # <zero-price-problem> # problem: a station is listed with 0 ISK price, solution: assume most frequent price instead # find most frequent bensin and diesel price price_frequency = {'bensin': {}, 'diesel': {}} for key in prices: # bensin bensin_price_key = str(prices[key]['bensin95']) if bensin_price_key not in price_frequency['bensin']: price_frequency['bensin'][bensin_price_key] = 0 price_frequency['bensin'][bensin_price_key] += 1 # diesel diesel_price_key = str(prices[key]['diesel']) if diesel_price_key not in price_frequency['diesel']: price_frequency['diesel'][diesel_price_key] = 0 price_frequency['diesel'][diesel_price_key] += 1 # most frequent prices most_frequent_bensin_price = float(max(price_frequency['bensin'])) most_frequent_diesel_price = float(max(price_frequency['diesel'])) for key in prices: if prices[key]['bensin95'] == 0.0: prices[key]['bensin95'] = most_frequent_bensin_price prices[key][ 'bensin95_discount'] = most_frequent_bensin_price - globs.N1_DISCOUNT if prices[key]['diesel'] == 0.0: prices[key]['diesel'] = most_frequent_diesel_price prices[key][ 'diesel_discount'] = most_frequent_diesel_price - globs.N1_DISCOUNT # </zero-price-problem> return prices