def mashov_main(**kw):
username, password, schoolID = kw.values()
default_headers = utils.headers(
referer="https://web.mashov.info/students/login")
session = requests.Session()
# login
resp = session.post(url="https://web.mashov.info/api/login",
headers=default_headers,
data=json.dumps(
dict(username=username,
password=password,
semel=schoolID,
year=datetime.now().year,
appName="info.mashov.students")))
csrf_token = resp.cookies.get('Csrf-Token')
login_js = resp.json()
utils.log(f'Logged in as {login_js["accessToken"]["displayName"]}',
src="mashov")
date = datetime.utcnow().isoformat().split('.')[0]
for student in login_js['accessToken']['children']:
utils.wait(2, add=2)
resp = session.put(
url="https://web.mashov.info/api/students/{0}/covid/{1}Z/clearance"
.format(student['childGuid'], date),
headers={
**default_headers, "Referer":
"https://web.mashov.info/students/main/covidClearance",
"X-Csrf-Token": csrf_token
},
data=json.dumps(
dict(
answer=3,
answererName=login_js["accessToken"]["displayName"],
answererId=login_js["credential"]["userId"],
clearanceDate=date,
lastAnswer=date,
lastAnswerIsOk=True,
studentClass='{classCode} {classNum}'.format_map(student),
studentName='{privateName} {familyName}'.format_map(
student),
userId=student['childGuid'],
noContactWithInfected=True,
noHeatAndSymptoms=True,
)))
if resp.ok:
utils.log(
f'The clearance was successfully signed for {student["privateName"]}',
src="mashov")
else:
utils.log(
f'Signature the clearance for {student["privateName"]} failed',
src="mashov")
def get_global_daelan_prices():
headers = utils.headers()
session = requests.Session()
res = session.get('https://daelan.is/', headers=headers)
html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
price_info_container = html.find('.//div[@id="gas-price-info-container"]')
bensin95 = None
diesel = None
for column in price_info_container.findall('.//li'):
if column.find('.//span').text == u'D\xedsel':
diesel_text = column.find('.//em').text.strip()
if diesel_text.endswith(' kr.'):
diesel_text = diesel_text[:-4]
diesel_text = diesel_text.replace(',', '.')
diesel = float(diesel_text)
elif column.find('.//span').text == u'Bens\xedn':
bensin95_text = column.find('.//em').text.strip()
if bensin95_text.endswith(' kr.'):
bensin95_text = bensin95_text[:-4]
bensin95_text = bensin95_text.replace(',', '.')
bensin95 = float(bensin95_text)
assert(bensin95 is not None)
assert(diesel is not None)
return {
'bensin95': bensin95,
'diesel': diesel,
# Dælan has no discount program
'bensin95_discount': None,
'diesel_discount': None
}
def get_individual_atlantsolia_prices():
relation = glob.ATLANTSOLIA_LOCATION_RELATION
url = 'https://www.atlantsolia.is/stodvaverd/'
res = requests.get(url, headers=utils.headers())
html_text = res.content
html = lxml.etree.fromstring(html_text, lxml.etree.HTMLParser())
div_prices = html.find(('.//*[@id="content"]/div/div/div/div[2]/div/div/'
'table/tbody'))
# Atlantsolia recently declared one of its stations to be without discount,
# on their site that station is marked with a star (*) and the following
# comment at the bottom:
# "* Engir afslaettir gilda"
discountless_stations = ('ao_008', )
prices = {}
for div_price in div_prices:
key = relation[div_price[0][0].text]
bensin95 = float(div_price[1][0].text.replace(',', '.'))
diesel = float(div_price[2][0].text.replace(',', '.'))
bensin95_discount = int(
(bensin95 - glob.ATLANTSOLIA_MINIMUM_DISCOUNT) * 10
) / 10.0
diesel_discount = int(
(diesel - glob.ATLANTSOLIA_MINIMUM_DISCOUNT) * 10
) / 10.0
if key in discountless_stations:
bensin95_discount = None
diesel_discount = None
prices[key] = {
'bensin95': bensin95,
'diesel': diesel,
'bensin95_discount': bensin95_discount,
'diesel_discount': diesel_discount
}
return prices
def get_offer(op:str,amount:str,isQuote:bool):
body = { 'op':op, 'amount':amount, 'isQuote':isQuote }
headers = utils.headers('offer', json.dumps(body))
response = requests.post(f'{BASE_URL}/offer', headers=headers, json=body).json()
if response == None or 'data' not in response:
raise Exception("Erro ao buscar oferta",response)
return response['data']
def _run(self, isbn):
# This dev key belongs to LibraryThing
url = 'http://www.librarything.com/devkey/3f8ba261818a718f42e1c94df68f48b8/large/isbn/%s' % isbn
if int(headers(url).get('content-length', 0)) > 100:
result = dict()
result['isbn'] = isbn
result['photo'] = url
result['source'] = "http://www.librarything.com/search.php?search=%s" % isbn
return result
else:
return None
def get_individual_ob_prices():
url = 'https://www.ob.is/eldsneytisverd/'
res = requests.get(url, headers=utils.headers(), verify=False)
html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
bensin95_text = html.find('.//*[@id="gas-price"]/span[1]').text
diesel_text = html.find('.//*[@id="gas-price"]/span[2]').text
bensin_discount_text = html.find('.//*[@id="gas-price"]/span[3]').text
diesel_discount_text = html.find('.//*[@id="gas-price"]/span[4]').text
bensin95 = float(bensin95_text.replace(',', '.'))
diesel = float(diesel_text.replace(',', '.'))
bensin95_discount = float(bensin_discount_text.replace(',', '.'))
diesel_discount = float(diesel_discount_text.replace(',', '.'))
prices = {}
ob_stations = utils.load_json(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'../stations/ob.json'
)
)
# 15 ISK discount for selected stations in September
#
# Occasionally, selected OB stations have had higher discount than usually
# for a specified month, it seems they're going to do this every month from
# now on (according to fb post in September).
# facebook.com/ob.bensin/photos/a.208957995809394/1904154726289704/
#
# Stations with additional discount in September 2018:
# * Baejarlind (ob_010)
# * Fjardarkaup (ob_012)
# * Njardvik (ob_024)
# * Starengi (ob_029)
# * Borgarnes (ob_009)
additional_discount = 15
additional_discount_stations = (
'ob_010', 'ob_012', 'ob_024', 'ob_029', 'ob_009',
)
now = datetime.datetime.now()
end = datetime.datetime.strptime('2018-09-30T23:59', '%Y-%m-%dT%H:%M')
for key in ob_stations:
bensin95_discount = bensin95_discount
diesel_discount = diesel_discount
if now < end and key in additional_discount_stations:
bensin95_discount = bensin95 - additional_discount
diesel_discount = diesel - additional_discount
prices[key] = {
'bensin95': bensin95,
'diesel': diesel,
'bensin95_discount': bensin95_discount,
'diesel_discount': diesel_discount
}
return prices
def get_global_olis_prices():
url = 'https://www.olis.is/solustadir/thjonustustodvar/eldsneytisverd/'
res = requests.get(url, headers=utils.headers(), verify=False)
html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
bensin95_text = html.find('.//*[@id="gas-price"]/span[1]').text
diesel_text = html.find('.//*[@id="gas-price"]/span[2]').text
bensin_discount_text = html.find('.//*[@id="gas-price"]/span[4]').text
diesel_discount_text = html.find('.//*[@id="gas-price"]/span[5]').text
return {
'bensin95': float(bensin95_text.replace(',', '.')),
'diesel': float(diesel_text.replace(',', '.')),
'bensin95_discount': float(bensin_discount_text.replace(',', '.')),
'diesel_discount': float(diesel_discount_text.replace(',', '.'))
}
def get_individual_orkan_prices():
# Read prices for Orkan and Orkan X stations because they're both on the
# same webpage.
url = 'https://www.orkan.is/orkan/orkustodvar/'
res = requests.get(url, headers=utils.headers(), verify=False)
html = lxml.etree.fromstring(res.content, lxml.etree.HTMLParser())
div_element = html.find('.//div[@class="accordion__container"]')
territories = div_element.findall('.//div[@class="accordion__child"]')
prices = {}
key = None
for territory in territories:
for station in territory:
station_name = station[0][0].text
bensin95 = float(station[1].text.replace(',', '.'))
diesel = float(station[2].text.replace(',', '.'))
if station_name in glob.ORKAN_LOCATION_RELATION:
key = glob.ORKAN_LOCATION_RELATION[station_name]
elif station_name in glob.ORKAN_X_LOCATION_RELATION:
key = glob.ORKAN_X_LOCATION_RELATION[station_name]
else:
continue
# Orkan X stations have key starting with "ox", while ordinary
# Orkan statins have keys starting with "or"
bensin95_discount = None
diesel_discount = None
if key.startswith('or'):
# Orkan has a 3-step discount system controlled by your
# spendings on gas from them the month before
# See more info here: https://www.orkan.is/Afslattarthrep
# For consistency we just use the minimum default discount
bensin95_discount = bensin95 - glob.ORKAN_MINIMUM_DISCOUNT
diesel_discount = diesel - glob.ORKAN_MINIMUM_DISCOUNT
prices[key] = {
'bensin95': bensin95,
'diesel': diesel,
'bensin95_discount': bensin95_discount,
'diesel_discount': diesel_discount
}
# TEMPORARY_FIX while three former Skeljungur stations are missing from the
# list of stations on Orkan webpage
for key in ['or_053', 'or_054', 'or_055']:
if key not in prices:
prices[key] = prices['or_000'].copy()
if key in ['or_054', 'or_055']:
# diesel currently 1 ISK more expensive on these two stations
prices[key]['diesel'] += 1
prices[key]['diesel_discount'] += 1
# /TEMPORARY_FIX
return prices
async def scrape_user_dir():
start_url = "https://twitter.com/i/directory/profiles/"
async with ClientSession() as session:
async with session.get(start_url,headers=headers()) as resp:
logger.info(f"[{resp.status}] {start_url}")
if resp.status > 400:
logger.warning(f"Bad response. Sleeping.")
await long_sleep()
return await asyncio.create_task(
scrape_user_dir())
await short_sleep()
html = await resp.text()
root = fromstring(html)
links = ['https://twitter.com'+str(link)
for link in root.xpath('//div[@class="character-links"]//a[@class="character"]/@href')]
for link in links:
await recurse_sub_dirs(link)
async def recurse_sub_dirs(url):
global file_rows
is_processed = await redis_client.sismember(proc_dir_pages, url)
if is_processed:
logger.info(f"Skipping processed directory page: {url}")
return
async with ClientSession() as session:
async with session.get(url,headers=headers()) as resp:
logger.info(f"[{resp.status}] {url}")
if resp.status > 400:
logger.warning(f"Bad response. Sleeping.")
await long_sleep()
return await asyncio.create_task(
recurse_sub_dirs(url))
await short_sleep()
html = await resp.text()
root = fromstring(html)
for link_ele in root.xpath('//div[@class="directory-page"]//div[@class="row"]//ul[@class="span3"]/li/a[@href]'):
link = 'https://twitter.com'+str(link_ele.xpath('./@href')[0])
name_ele = link_ele.xpath('./span[@class="name"]/text()')
screenname_ele = link_ele.xpath('./span[@class="screenname"]/text()')
if name_ele and screenname_ele:
name = str(name_ele[0]).strip()
screenname = str(screenname_ele[0]).strip()
logger.info(f"Extracted profile: {name}, {screenname}, {link}")
# cache url in redis.
data = json.dumps({
'name': name,
'screenname': screenname,
'url': link})
await redis_client.sadd(prof_data_key, data)
# mark this directory page as complete.
await redis_client.sadd(proc_dir_pages, data)
else:
logger.info(f"Extracted directory link: {link}")
await asyncio.create_task(
recurse_sub_dirs(link))
def get_global_n1_prices():
url_eldsneyti = 'https://www.n1.is/thjonusta/eldsneyti'
url_eldsneyti_api = 'https://www.n1.is/umbraco/api/fuel/GetSingleFuelPrice'
headers = utils.headers()
session = requests.Session()
session.get(url_eldsneyti, headers=headers)
post_data_bensin = 'fuelType=95+Oktan'
post_data_diesel = 'fuelType=D%C3%ADsel'
headers_eldsneyti_api = {
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-US,en;q=0.8,is;q=0.6',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Host': 'www.n1.is',
'Origin': 'https://www.n1.is',
'Referer': 'https://www.n1.is/eldsneyti/',
'User-Agent': headers['User-Agent'],
'X-Requested-With': 'XMLHttpRequest'
}
headers_eldsneyti_api['Content-Length'] = str(len(post_data_bensin))
res = session.post(url_eldsneyti_api, data=post_data_bensin,
headers=headers_eldsneyti_api)
bensin95_discount_text = res.content
headers_eldsneyti_api['Content-Length'] = str(len(post_data_diesel))
res = session.post(url_eldsneyti_api, data=post_data_diesel,
headers=headers_eldsneyti_api)
diesel_discount_text = res.content
prices = {}
prices['bensin95'] = float(
bensin95_discount_text.replace('"', '').replace(',', '.'))
prices['diesel'] = float(
diesel_discount_text.replace('"', '').replace(',', '.'))
prices['bensin95_discount'] = prices['bensin95'] - glob.N1_DISCOUNT
prices['diesel_discount'] = prices['diesel'] - glob.N1_DISCOUNT
return prices
def confirm_offer(offerId:str):
body = { 'offerId':offerId }
headers = utils.headers('offer/confirm', json.dumps(body))
response = requests.post(f'{BASE_URL}/offer/confirm',headers=headers,json=body).json()
return response
def get_balance():
headers = utils.headers('balance')
response = requests.post(f'{BASE_URL}/balance', headers=headers).json()
return response