def get_jam_tracks_threads(one_date):
data = {
'day': one_date.day,
'month': one_date.month,
'hour': one_date.hour,
'minute': one_date.minute
}
res = req_post(self.tracks_request_url, data=data)
responds_list.append((res.status_code, res.text))
elements = BeautifulSoup(res.text, 'html.parser').find_all('li')
for li in elements:
track_str = li.text
if 'Cinema' not in track_str and 'Ledjam Radio' not in track_str and 'Television' not in track_str:
splited = track_str.split(' : ')
hours, minutes = splited[0].split('h')
track_play_date = datetime(day=one_date.day,
month=one_date.month,
year=one_date.year,
hour=int(hours),
minute=int(minutes))
common_name = ':'.join(splited[1:]).split('-')
title = common_name[0].strip()
artist = ('-'.join(common_name[1:])).strip()
common_name = f'{artist} - {title}'
to_add = {
'artist': artist,
'title': title,
'play_date': track_play_date,
'common_name': common_name,
'genre': 'djam'
}
if to_add not in res_list:
res_list.append(to_add)
def post(self,
url,
data,
timeout=60,
headers={},
verify=False,
type='python'):
url = self.quote(url)
if type == 'python':
try:
from requests import post as req_post
return req_post(url,
data,
timeout=timeout,
headers=headers,
verify=verify)
except:
if sys.version_info[0] == 2:
result = self._post_py2(url, data, timeout, headers,
verify)
else:
result = self._post_py3(url, data, timeout, headers,
verify)
elif type == 'curl':
result = self._post_curl(url, data, timeout, headers, verify)
elif type == 'php':
result = self._post_php(url, data, timeout, headers, verify)
return result
def post(self,
url,
data,
timeout=60,
headers={},
verify=False,
type='python'):
url = self.quote(url)
if type == 'python':
try:
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
from requests import post as req_post
return req_post(url,
data,
timeout=timeout,
headers=headers,
verify=verify)
except:
result = self._post_curl(url, data, timeout, headers, verify)
elif type == 'curl':
result = self._post_curl(url, data, timeout, headers, verify)
elif type == 'php':
result = self._post_php(url, data, timeout, headers, verify)
elif type == 'src':
if sys.version_info[0] == 2:
result = self._post_py2(url, data, timeout, headers, verify)
else:
result = self._post_py3(url, data, timeout, headers, verify)
return result
def spotify_callback():
auth_token = request.args['code']
code_payload = {
'grant_type': 'authorization_code',
'code': str(auth_token),
'redirect_uri': url_for('oauth.spotify_callback', _external=True),
'client_id': SpotifyConfig.CLIENT_ID,
'client_secret': SpotifyConfig.CLIENT_KEY,
}
post_request = req_post(SpotifyConfig.SPOTIFY_TOKEN_URL, data=code_payload)
if post_request.status_code != 200 or 'token' not in post_request.text:
return {'success': False, 'result': post_request.text}
response_data = json_loads(post_request.text)
sp = Spotify(token=response_data['access_token'],
expires_in_sec=response_data['expires_in'])
user_data = sp.get_user_info()
if not user_data['success']:
return {'success': False, 'result': 'Failed to get user data'}
session['logged_in'] = True
session['ms_user'] = user_data['result']
session['ms_service'] = 'spotify'
response_data['token'] = response_data['access_token']
del response_data['access_token']
session['oauth'] = response_data
session['oauth']['expiration_time'] = datetime.now() + timedelta(
seconds=response_data['expires_in'])
return redirect(url_for('oauth.set_cookies'))
def deezer_callback():
auth_token = request.args['code']
code_payload = {
'code': str(auth_token),
'app_id': DeezerConfig.CLIENT_ID,
'secret': DeezerConfig.CLIENT_SECRET,
}
post_request = req_post(DeezerConfig.DEEZER_TOKEN_URL, data=code_payload)
if post_request.status_code != 200 or 'token' not in post_request.text:
return {'success': False, 'result': post_request.text}
token, expires_in = post_request.text.split('&')
token = token.split('token=')[-1]
expires_in = int(expires_in.split('expires=')[-1])
dz = Deezer(token, expires_in)
user_data = dz.get_user_info()
if not user_data['success']:
return {'success': False, 'result': 'Failed to get user data'}
session['logged_in'] = True
session['ms_user'] = user_data['result']
session['ms_service'] = 'deezer'
session['oauth'] = {
'token': token,
'expiration_time': datetime.now() + timedelta(seconds=expires_in)
}
return redirect(url_for('oauth.set_cookies'))
def get_latest_tracks(self):
end_time = datetime.now()
orig_date = end_time.strftime('%d-%m-%Y')
start_time = end_time - timedelta(minutes=30)
start_time = str(int(start_time.timestamp()))
end_time = str(int(end_time.timestamp()))
res_list = []
genre = self.radio_id.split('_')[-1].lower()
url = self.tracks_request_url
json = {
'query':
'{ grid'
f'(start: {start_time}, end: {end_time}, station: {self.radio_id}) '
'{ ... on TrackStep { start track { mainArtists title albumTitle } } } } '
}
headers = {'x-token': APIConfig.radiofrance_api_key}
res = req_post(url=url, json=json, headers=headers)
if res.status_code != 200:
return {
'success': False,
'result': 'Failed to get tracks from API',
'respond': ''
}
tracks = json_loads(res.text)['data']['grid']
for track in tracks:
play_time = track.get('start', '')
if not play_time:
play_date = orig_date
else:
play_date = datetime.fromtimestamp(play_time).strftime(
'%d/%m/%Y %H:%M:%S')
track = track['track']
if track:
artist = ','.join(track.get('mainArtists', '')).strip()
title = track.get('title', '').strip()
common_name = f'{artist} - {title}'
res_list.append({
'album_name': track.get('albumTitle', ''),
'common_name': common_name,
'artist': artist,
'title': title,
'play_date': play_date,
'radio_name': self.radio_id,
'genre': genre
})
return {'success': True, 'result': res_list, 'respond': ''}
def upload_haven(link_to_download):
file_storage = new_soup(get_html(link_to_download).content)
form = file_storage.find_all('form', method='POST')[0]
form_all_inputs = form.find_all('input')
all_inputs = {
'_token': form_all_inputs[0].get('value').strip(),
'key': form_all_inputs[1].get('value').strip(),
'time': form_all_inputs[2].get('value').strip(),
'hash': form_all_inputs[2].get('value').strip()
}
time_sleep(6)
result_form = req_post(link_to_download,
headers=headers,
data=all_inputs,
params=all_inputs,
timeout=2.50)
fast_write(result_form.text, 'test.html')
def get_radios(cls):
json = {
'query':
'{ brand'
f'(id: {cls.parent_id}) '
'{ id liveStream webRadios { id liveStream description} } } '
}
headers = {'x-token': APIConfig.radiofrance_api_key}
res = req_post(url=cls.tracks_request_url, json=json, headers=headers)
if res.status_code != 200:
return {'success': False, 'result': res.text}
radio_url = cls.url
radio = json_loads(res.text)['data']['brand']
radios = radio['webRadios']
fip_radio = {
'name': radio['id'],
'url': radio_url,
'stream_url': radio['liveStream'],
'description': cls.description,
'genre': cls.genre,
'country': cls.country
}
fip_stations = {}
for radio in radios:
fip_stations[radio['id']] = {
'name': radio['id'],
'url': radio_url,
'stream_url': radio['liveStream'],
'genre': radio['id'].split('_')[-1],
'description': radio['description'],
'country': cls.country
}
fip_stations[fip_radio['name']] = fip_radio
for excluded_station in cls.exclude_radios:
if excluded_station in fip_stations:
del fip_stations[excluded_station]
return {
'success': len(fip_stations) > 0,
'result': fip_stations,
'respond': ''
}
def get_user_data(self, code_data):
print('get_user_data')
print(code_data)
serializer = UserUAoauthSerializer(data=code_data)
serializer.is_valid(raise_exception=True)
data = {
"access_token": serializer.validated_data['access_token'],
"user_id": serializer.validated_data['user_id']
}
res = req_post(USER_INFO_URL, data=data)
print(USER_INFO_URL + '?=' + "access_token=" +
serializer.validated_data['access_token'] + '&'
"user_id=" + serializer.validated_data['user_id'])
if res.status_code == 200:
return res.json()
elif res.status_code == 401:
raise ValidationError(res.json())
def get_tracks_by_threads(one_date):
start_time = str(int(one_date.timestamp()))
end_time = str(
int((one_date + timedelta(hours=time_step)).timestamp()))
genre = self.radio_id.split('_')[-1].lower()
url = self.tracks_request_url
json = {
'query':
'{ grid'
f'(start: {start_time}, end: {end_time}, station: {self.radio_id}) '
'{ ... on TrackStep { start track { mainArtists title albumTitle } } } } '
}
headers = {'x-token': APIConfig.radiofrance_api_key}
res = req_post(url=url, json=json, headers=headers)
responds_list.append((res.status_code, res.text))
if res.status_code != 200:
return {
'success': False,
'result': 'Failed to get tracks from API',
'respond': ''
}
tracks = json_loads(res.text)['data']['grid']
for track in tracks:
play_time = track.get('start', '')
track_play_date = datetime.fromtimestamp(play_time)
track = track['track']
if track:
artist = ','.join(track.get('mainArtists', '')).strip()
title = track.get('title', '').strip()
common_name = f'{artist} - {title}'
to_add = {
'album_name': track.get('albumTitle', ''),
'common_name': common_name,
'artist': artist,
'title': title,
'play_date': track_play_date,
'radio_name': self.radio_id,
'genre': genre
}
if to_add not in res_list:
res_list.append(to_add)
def get_latest_tracks(self):
"""
0 tracks - current track
∟ timetoplay - time after which need to make next request to get new current playing
other tracks - previous played (9 tracks)
"""
latest_tracks = []
res = req_post(self.current_playing_url, data={'origin': 'website'})
res = json_loads(res.text)['tracks']
for one_track in res:
seconds_to_play = one_track.get('timetoplay', 0) + 25
seconds_playing = int(one_track['duration']) - seconds_to_play
start_date = datetime.now() - timedelta(seconds=seconds_playing)
end_date = datetime.now() + timedelta(seconds=seconds_to_play)
spotify = one_track['spotify']
if spotify:
spotify = spotify.get('id', '').split('track:')[-1]
deezer = one_track['deezer']
if deezer:
deezer = deezer.get('id', '')
track_dict = {
'artist': one_track['artist'],
'title': one_track['title'],
'common_name': f"{one_track['artist']} - {one_track['title']}",
'timetoplay': seconds_to_play,
'spotify': spotify,
'deezer': deezer,
'radio_name': self.radio_id,
'start_date': start_date,
'end_date': end_date
}
latest_tracks.append(track_dict)
return latest_tracks
def get_acess_token(self, request):
print('get_acess_token')
serializer = BearesSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
##self.redirect_uri = serializer.validated_data['redirect_uri'],
data = {
"grant_type": "authorization_code",
"client_id": CLIENT_ID,
"client_secret": CLIENT_SECRET,
"code": serializer.validated_data['code'],
##"redirect_uri": self.redirect_uri
}
try:
res = req_post(ACCESS_TOKEN_URL, data=data)
except exceptions.ConnectionError as e:
raise ValidationError({
'non_field_errors':
'Не вдалось підєднатись до серверу автоизації',
'details': str(e)
})
if res.status_code == 200:
return res.json()
elif res.status_code == 401:
raise ValidationError(res.json())
def parser(original):
debug(original, 'Parse Original Message:')
# message = spell_checker(original)
# if message:
message = [original]
debug('Message After spell check: {}'.format(message))
tmpslot = dict(
originalText=[],
lemma=[],
pos=[],
slot_tag=[],
)
try:
slot_response = req_post('http://13.228.72.161:8080/tag',
json={
"q": message
}).json()
general_slots = slot_response.get('result') or [('LEXICA_SLOT', ) * 4]
debug('general_slots: {}'.format(general_slots))
(tmpslot['originalText'], tmpslot['lemma'], tmpslot['pos'],
tmpslot['slot_tag']) = zip(*general_slots)
except JSONDecodeError as err:
debug('JSON error [{}]'.format(err), 'ERROR')
message = ' '.join(message)
predict_result = dict(
zip(('label', 'original', 'stanford'), predict(message)))
debug(predict_result, name='Predict Result:')
# predict_result['stanford'] = predict_result.get('stanford')[0]
label_score = float(predict_result.get('label')[0].get('score'))
original_score = float(predict_result.get('original')[0].get('score'))
if label_score > original_score:
class_probability = predict_result.get('label')
else:
class_probability = predict_result.get('original')
del label_score, original_score, predict_result
words = list(
filter(
None,
[word.strip() for word in sub(r'[^\w\s]', ' ', message).split()]))
# create new words for BookSlot_MEMM prevent mutation of words
book_slot = BookSlot_MEMM(words[:])
general_slots = list(map(list, zip(*list(tmpslot.values()))))
book_keyword = ''
for i, val in enumerate(words):
if book_slot[i] == 'book':
book_keyword += ' ' + val
book_keyword = book_keyword.strip() if book_keyword.strip() else message
debug(book_slot, name='book_slot: ')
debug(book_keyword, name='book_keyword: ')
debug(class_probability, name='class_probability: ')
debug(general_slots, name='general_slots: ')
debug(words, name='words: ')
return message, book_keyword, class_probability, general_slots, words
def get_current_track(self):
# orig_date = datetime.now().strftime('%d/%m/%Y %H:%M:%S')
url = self.tracks_request_url
json = {
'query':
'{ live'
f'(station: {self.radio_id}) '
'{ song { start end track { mainArtists title } } } } '
}
headers = {'x-token': APIConfig.radiofrance_api_key}
res = req_post(url=url, json=json, headers=headers)
if res.status_code != 200:
return {
'success': False,
'result': 'Failed to get tracks from API',
'respond': ''
}
track = json_loads(res.text)['data']['live']['song']
if not track:
now = datetime.now()
return {
'common_name': 'unknown - unknown',
'artist': 'unknown',
'title': 'unknown',
'start_date': now,
'end_date': now + timedelta(seconds=30),
'radio_name': self.radio_id,
'timetoplay': 30,
'spotify': '',
'deezer': ''
}
start_time = track.get('start', '')
start_date = datetime.fromtimestamp(start_time)
end_time = track.get('end', '')
end_date = datetime.fromtimestamp(end_time)
timetoplay = (end_date - datetime.now()).seconds
track = track['track']
if track:
artist = ','.join(track.get('mainArtists', '')).strip()
title = track.get('title', '').strip()
common_name = f'{artist} - {title}'
return {
'common_name': common_name,
'artist': artist,
'title': title,
'start_date': start_date,
'end_date': end_date,
'radio_name': self.radio_id,
'timetoplay': timetoplay,
'spotify': '',
'deezer': ''
}
return {}
def parse(self, response):
meta = response.meta
item = meta['item']
username = item["username"]
try:
cookies_dict = dict()
req_ss = req_session()
index_res = req_ss.get(self._start_url_,
headers=self.headers,
verify=False)
cookies_dict.update(index_res.cookies.get_dict())
# 获取cookies
index_cookie_url = 'https://v4.passport.sohu.com/i/cookie/common' \
'?callback=passport403_cb%s&_=%s' % (get_js_time(), get_js_time())
index_cookie_res = req_ss.get(index_cookie_url, verify=False)
cookies_dict.update(index_cookie_res.cookies.get_dict())
code_callback_url = 'https://v4.passport.sohu.com/i/jf/code?callback=passport403_cb%s' \
'&type=0&_=%s' % (get_js_time(), get_js_time())
code_callback_res = req_ss.get(code_callback_url, verify=False)
cookies_dict.update(code_callback_res.cookies.get_dict())
# 解析
jv_val = self.js_driver(code_callback_res.text).split('=')
jv_dict = {jv_val[0]: jv_val[1].split(';', 1)[0]}
req_ss.cookies.update(jv_dict)
cookies_dict.update(jv_dict)
data = {
'userid': username,
'password': md5(item["password"].encode('utf-8')).hexdigest(),
'appid': '101305',
'callback': 'passport403_cb' + get_js_time()
}
login_url = 'https://v4.passport.sohu.com/i/login/101305'
res = req_ss.post(login_url, data=data)
result = json_loads(self.result_pattern2.search(res.text).group(1))
status = result['status']
if status == 404:
yield from self.error_handle(username,
"搜狐邮箱---账号密码错误",
tell_msg="请刷新页面重试")
return
elif status == 465:
callback_url = 'https://v4.passport.sohu.com/i/jf/code?callback=passport403_cb%s' \
'&type=0&_=%s' % (get_js_time(), get_js_time())
jv_content = req_ss.get(callback_url)
cookies_dict.update(jv_content.cookies.get_dict())
code_callback_res = req_ss.get(code_callback_url, verify=False)
jv = self.js_driver(code_callback_res.text).split('=')
cookies_dict.update({jv[0]: jv[1].split(';', 1)[0]})
pagetoken = get_js_time()
captcha_url = 'https://v4.passport.sohu.com/i/captcha/picture?pagetoken=%s' \
'&random=passport403_sdk%s' % (pagetoken, get_js_time())
self.set_image_captcha_headers_to_ssdb(cookies_dict, username)
self.set_email_img_url_to_ssdb(captcha_url, username)
captcha_body = req_get(captcha_url,
cookies=cookies_dict).content
captcha_code = self.ask_image_captcha(captcha_body,
username,
file_type=".png")
data.update({
'captcha': captcha_code,
'pagetoken': str(pagetoken)
})
res = req_post(login_url,
data=data,
cookies=cookies_dict,
verify=False)
result_two = json_loads(
self.result_pattern2.search(res.text).group(1))
status_two = result_two['status']
if status_two == 420:
yield from self.error_handle(username,
"搜狐邮箱---输入验证错误!",
tell_msg="输入验证错误!")
return
elif status_two != 200:
yield from self.error_handle(username,
"搜狐邮箱---未知错误!",
tell_msg="抓取失败!")
return
if res:
# 登录成功之后需发送回滚请求,
cookies_dict.update(res.cookies.get_dict())
callback_url = 'https://mail.sohu.com/fe/login/callback'
call_back = req_ss.post(callback_url, verify=False)
# 更新回滚 cookie
cookies_dict.update(call_back.cookies.get_dict())
# 构造登出cookies
logout_con = req_ss.get(
'https://v4.passport.sohu.com/i/jf/code?callback=passport403_cb%s'
'&type=0&_=%s' % (get_js_time(), get_js_time()),
cookies=cookies_dict,
verify=False)
ppmdig_cookies = logout_con.cookies.get_dict()
logout_val = self.js_driver(logout_con.text).split('=')
ppmdig_cookies.update(
{logout_val[0]: logout_val[1].split(';', 1)[0]})
search_url = self.search_url % (0, get_js_time(), self.keyword)
meta["cookies_dict"] = cookies_dict
meta["ppmdig_cookies"] = ppmdig_cookies
yield Request(url=search_url,
cookies=cookies_dict,
callback=self.parse_search,
meta=meta,
dont_filter=True,
errback=self.err_callback)
# 登录成功之后直接访问json页面,提取对应数据进行筛选
# 登录成功后,用户用可能讲账单添加到自定义文件夹里,这里需要筛选用户自定义内容,自定的新ID为17开始
# 首先判断是否存在自定义标签
else:
yield from self.error_handle(username,
"搜狐邮箱---账号密码错误!",
tell_msg="账号或密码错误,请刷新页面重试")
except Exception:
yield from self.except_handle(username,
"登录入口异常",
tell_msg="邮箱登录失败，请刷新页面重试")
def board_display(board):
global BOARDS
if board == "favicon.ico":
return render_template("404.html"), 404
with open(ROOT + "boardlist") as f:
BOARDS = load(f)
if board in BOARDS:
desc = BOARDS[board]
with open(ROOT + "boards/{}/index".format(board)) as f:
board_content = load(f)
else:
return render_template("404.html"), 404
if request.method == "POST":
global POSTS
title = request.form["title"] if request.form.get("title", None) else ""
name = "Anonymous"
body = request.form["body"] if request.form.get("body", None) else ""
ip = request.headers.get("X-Forwarded-For", None)
logging.info(ip)
if not request.form.get("g-recaptcha-response", None):
flash("You'll need to do something with the captcha please.", "error")
return render_template("posted.html", board=board, desc=desc)
try:
with open("/home/blha303/recaptchakey") as f:
captcha_resp = req_post("https://www.google.com/recaptcha/api/siteverify", data={"secret": f.read().strip(), "response": request.form.get("g-recaptcha-response"), "remoteip": ip}).json()
if not captcha_resp["success"]:
flash("Your captcha wasn't up to par. Want to try again? (You'll have to retype your message, sorry.)", "error")
return render_template("posted.html", board=board, desc=desc)
except:
logging.exception("Error with captcha")
flash("Sorry, there was a problem while processing the captcha. Please <a href='https://twitter.com/blha303'>let me know</a>.", "error")
return render_template("posted.html", board=board, desc=desc)
if '<div ' in body:
flash("Hi there. Sorry to rain on your parade, but I can't let you do that. Soz.", "error")
return render_template("posted.html", board=board, desc=desc)
if clean_html(html(body).strip()) == "<div></div>":
body = ""
id = request.form["id"] if request.form.get("id", None) else ""
if len(body) > 1500 or len(title) > 30 or len(name) > 30:
flash("Too long! Want to try that again?", "error")
return render_template("posted.html", board=board, desc=desc)
if not body:
flash("No body provided! We kinda need something there, sorry.", "error")
return render_template("posted.html", board=board, desc=desc)
if not board in POSTS:
POSTS[board] = 1
if id and id.isdigit():
changed_something = False
for post in board_content:
if post[0] == int(id):
POSTS[board] += 1
post.append([int(datetime.timestamp(datetime.utcnow())),
POSTS[board] + 1,
body])
changed_something = True
if not changed_something:
flash("Sorry, couldn't find that top-level post.", "error")
return render_template("posted.html", board=board, desc=desc)
else:
POSTS[board] += 1
board_content.append([POSTS[board] + 1,
title,
[int(datetime.timestamp(datetime.utcnow())),
POSTS[board] + 1,
body]
])
with open(ROOT + "postnums", "w") as f:
dump(POSTS, f)
with open(ROOT + "boards/{}/index".format(board), "w") as f:
dump(board_content, f)
flash("Success! (?)", "success")
return render_template("posted.html", board=board, desc=desc, id=POSTS[board])
toplevel = process_board(board_content)
return render_template("board.html", posts=toplevel, board=board, desc=desc)
