def _extract_content_as_dict(url):
def remove_unnecessary_info_from_json_dict(json_dict):
trimmed_dict = dict()
for key, value in json_dict.items():
if value and key in _info_to_crawl:
trimmed_dict[key] = value
return trimmed_dict
url, attributes = _parse_and_redirect_url(url)
soup = get_soup(url)
if 'sports' in url:
json_dict = _parse_sport(soup)
elif 'entertain' in url:
json_dict = _parse_entertain(soup)
else:
json_dict = _parse_basic(soup)
json_dict.update(attributes)
json_dict.update({
'url': url,
'crawlerVersion': version,
'crawledTime': current_timestamp()
})
json_dict = remove_unnecessary_info_from_json_dict(json_dict)
return json_dict
def end_session(session_id):
connection, cursor = utils.connect_to_db("study")
query_string = """UPDATE sessions SET session_end = %s WHERE session_id = %s;"""
data = (utils.current_timestamp(), session_id)
cursor.execute(query_string, data)
connection.commit()
async def find_offline_users(session):
stmt = select(User).where(User.last_seen_at < current_timestamp() - 60,
User.online == True)
async with session.begin():
result = await session.execute(stmt)
users = result.scalars().all()
for user in users:
user.online = False
session.add(user)
def start_session(session_id, ip_address):
connection, cursor = utils.connect_to_db("study")
query_string = """INSERT INTO sessions
(session_start, session_id, ip_address)
VALUES (%s, %s, %s)
ON CONFLICT DO NOTHING;"""
data = (utils.current_timestamp(), session_id, ip_address)
cursor.execute(query_string, data)
connection.commit()
import requests
from utils import current_timestamp
headers = {
'Referer': 'http://iservice.10010.com/e4/skip.html?menuCode=000100020001&menuCode=000100020001',
'X-Requested-With': 'XMLHttpRequest',
}
cookies = {
'_uop_id': '6e6081667d1dcb1ffb345c4904702c70',
'JUT': 'UUO3nPc+pShsFLFD8UYOe6xvWUP6OuIlpknMPAE1rKKokHhCq3uCY+mebrcCSefac0G4xTriaw4hiwcDAqbdHwjZ1Ni8Ln9eLfi+8osYtm7bfPe+jW/q5tB9MGgbe/XShlLEORoWNCPPkigd5KIZ8nfzE1qwYfCkcxTyJt14OOPf9mSnSRqq4U6PGS6XBEA2p7xiuD9IL0qIph3hXhKUhBe0sIFIDJlx5LyjvzE9aYEoB5YRl0VACzcnBEG6GsguYeNn02FeI10xYVsE6FoV+lUShH5TmPtvw73Vl9OsZwT60EHx7IZ6O9LkVtXR8fVb7AzG7CK1TAPUIqZeS8m8sBdhPtFEs1BsY6neQjwHfUBO8mNZ1lrz3JiNjQ3RylXvJjQFPUhmHYKCE/ciJrv4qstCg5Lo5e+gAYB/bjqvEI9CfvXHb5Iecj6GZM+D/hTfx6qwvZUmlpBvGjRHeatl/o4XXCc9qhVxrBS8RwZaorE4WKdi4K7r1JfGGRNPIRUmTHzGr6eXSAbRlkGl56ZY1Q==LCGDhXtevKd81/lBF0fWIQ=='
}
resp = requests.post('http://iservice.10010.com/e3/static/check/checklogin/', params={'_': current_timestamp()}, headers=headers, cookies=cookies)
print(resp.status_code)
print(resp.text)
print(resp.cookies)
import requests
from utils import current_timestamp
oauth_url = 'https://uac.10010.com/oauth2/genqr?timestamp={}'.format(
current_timestamp())
resp = requests.get(oauth_url)
print(resp.cookies.__dict__)
# def pre_login2(self):
# oauth_url = 'https://uac.10010.com/oauth2/genqr?timestamp={}'.format(current_timestamp())
#
# check_need_verify_url = 'https://uac.10010.com/portal/Service/CheckNeedVerify'
# check_need_verify_headers = {
# 'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
# 'Accept-Encoding': 'gzip, deflate, br',
# 'X-Requested-With': 'XMLHttpRequest',
# 'Referer': 'https://uac.10010.com/portal/homeLogin',
# 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) '
# 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
# }
# check_need_verify_cookies = {
#
# }
# check_need_verify_params = {
# 'callback': '',
# 'userName': self.mobile,
# 'pwdType': '01',
# '_': str(current_timestamp())
# }
def ping(self):
self.last_seen_at = current_timestamp()
self.online = True
def send_update_to_user(user: str, auto, codes=None):
user_data = None
try:
user_data = persistence.user_data[user]
running = user_data.setdefault(Section.Running.value, False)
update_updater_data()
if Section.API_Key.value not in user_data:
updater.bot.send_message(
user, text='API Key not set ⛔️\nSet in /settings')
return
if running and not auto:
updater.bot.send_message(user, text='Already running 🏃')
return
user_data[Section.Running.value] = True
market = Market(user_data[Section.API_Key.value])
now = current_timestamp()
first = True
for code in codes or user_data.get(Section.Watchlist.value, {}):
try:
code_data = persistence.user_data[user][
Section.Watchlist.value][code]['value']
last_run = code_data[Section.LastRun.value]
frequency = parse(code_data[Section.Frequency.value])
interval = parse(code_data[Section.Interval.value])
enabled = code_data[Section.Enabled.value]
# print(code, last_run + frequency, now, last_run + frequency - now)
if auto and (not enabled or last_run + frequency > now):
continue
code_data[Section.LastRun.value] = current_timestamp()
update_updater_data()
if first:
if auto:
updater.bot.send_message(user,
text='Getting updates 🌎')
first = False
else:
# Wait to not overload the api
msg = updater.bot.send_message(
user, text='Waiting 60 seconds for API... ⏳')
run(sleep(60))
msg.delete()
msg = updater.bot.send_message(user,
text=f'Calculating {code}... ⏳')
delta = datetime.fromtimestamp(now - interval)
chart = market.get_wma(code, delta)
msg.delete()
updater.bot.send_photo(
user,
photo=chart,
disable_notification=True,
caption=f'{code} - {code_data[Section.Interval.value]}')
except Exception as e:
print(f'❌ {user} - {e}')
updater.bot.send_message(user,
text=f'There was an error ⚠️\n {e}')
if not first and auto: # Has run at least once
updater.bot.send_message(user, text=f'Done ✅')
finally:
if user_data:
user_data[Section.Running.value] = False
update_updater_data()
