def get_jam_tracks_threads(one_date): data = { 'day': one_date.day, 'month': one_date.month, 'hour': one_date.hour, 'minute': one_date.minute } res = req_post(self.tracks_request_url, data=data) responds_list.append((res.status_code, res.text)) elements = BeautifulSoup(res.text, 'html.parser').find_all('li') for li in elements: track_str = li.text if 'Cinema' not in track_str and 'Ledjam Radio' not in track_str and 'Television' not in track_str: splited = track_str.split(' : ') hours, minutes = splited[0].split('h') track_play_date = datetime(day=one_date.day, month=one_date.month, year=one_date.year, hour=int(hours), minute=int(minutes)) common_name = ':'.join(splited[1:]).split('-') title = common_name[0].strip() artist = ('-'.join(common_name[1:])).strip() common_name = f'{artist} - {title}' to_add = { 'artist': artist, 'title': title, 'play_date': track_play_date, 'common_name': common_name, 'genre': 'djam' } if to_add not in res_list: res_list.append(to_add)
def post(self, url, data, timeout=60, headers={}, verify=False, type='python'): url = self.quote(url) if type == 'python': try: from requests import post as req_post return req_post(url, data, timeout=timeout, headers=headers, verify=verify) except: if sys.version_info[0] == 2: result = self._post_py2(url, data, timeout, headers, verify) else: result = self._post_py3(url, data, timeout, headers, verify) elif type == 'curl': result = self._post_curl(url, data, timeout, headers, verify) elif type == 'php': result = self._post_php(url, data, timeout, headers, verify) return result
def post(self, url, data, timeout=60, headers={}, verify=False, type='python'): url = self.quote(url) if type == 'python': try: import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings( InsecureRequestWarning) from requests import post as req_post return req_post(url, data, timeout=timeout, headers=headers, verify=verify) except: result = self._post_curl(url, data, timeout, headers, verify) elif type == 'curl': result = self._post_curl(url, data, timeout, headers, verify) elif type == 'php': result = self._post_php(url, data, timeout, headers, verify) elif type == 'src': if sys.version_info[0] == 2: result = self._post_py2(url, data, timeout, headers, verify) else: result = self._post_py3(url, data, timeout, headers, verify) return result
def spotify_callback(): auth_token = request.args['code'] code_payload = { 'grant_type': 'authorization_code', 'code': str(auth_token), 'redirect_uri': url_for('oauth.spotify_callback', _external=True), 'client_id': SpotifyConfig.CLIENT_ID, 'client_secret': SpotifyConfig.CLIENT_KEY, } post_request = req_post(SpotifyConfig.SPOTIFY_TOKEN_URL, data=code_payload) if post_request.status_code != 200 or 'token' not in post_request.text: return {'success': False, 'result': post_request.text} response_data = json_loads(post_request.text) sp = Spotify(token=response_data['access_token'], expires_in_sec=response_data['expires_in']) user_data = sp.get_user_info() if not user_data['success']: return {'success': False, 'result': 'Failed to get user data'} session['logged_in'] = True session['ms_user'] = user_data['result'] session['ms_service'] = 'spotify' response_data['token'] = response_data['access_token'] del response_data['access_token'] session['oauth'] = response_data session['oauth']['expiration_time'] = datetime.now() + timedelta( seconds=response_data['expires_in']) return redirect(url_for('oauth.set_cookies'))
def deezer_callback(): auth_token = request.args['code'] code_payload = { 'code': str(auth_token), 'app_id': DeezerConfig.CLIENT_ID, 'secret': DeezerConfig.CLIENT_SECRET, } post_request = req_post(DeezerConfig.DEEZER_TOKEN_URL, data=code_payload) if post_request.status_code != 200 or 'token' not in post_request.text: return {'success': False, 'result': post_request.text} token, expires_in = post_request.text.split('&') token = token.split('token=')[-1] expires_in = int(expires_in.split('expires=')[-1]) dz = Deezer(token, expires_in) user_data = dz.get_user_info() if not user_data['success']: return {'success': False, 'result': 'Failed to get user data'} session['logged_in'] = True session['ms_user'] = user_data['result'] session['ms_service'] = 'deezer' session['oauth'] = { 'token': token, 'expiration_time': datetime.now() + timedelta(seconds=expires_in) } return redirect(url_for('oauth.set_cookies'))
def get_latest_tracks(self): end_time = datetime.now() orig_date = end_time.strftime('%d-%m-%Y') start_time = end_time - timedelta(minutes=30) start_time = str(int(start_time.timestamp())) end_time = str(int(end_time.timestamp())) res_list = [] genre = self.radio_id.split('_')[-1].lower() url = self.tracks_request_url json = { 'query': '{ grid' f'(start: {start_time}, end: {end_time}, station: {self.radio_id}) ' '{ ... on TrackStep { start track { mainArtists title albumTitle } } } } ' } headers = {'x-token': APIConfig.radiofrance_api_key} res = req_post(url=url, json=json, headers=headers) if res.status_code != 200: return { 'success': False, 'result': 'Failed to get tracks from API', 'respond': '' } tracks = json_loads(res.text)['data']['grid'] for track in tracks: play_time = track.get('start', '') if not play_time: play_date = orig_date else: play_date = datetime.fromtimestamp(play_time).strftime( '%d/%m/%Y %H:%M:%S') track = track['track'] if track: artist = ','.join(track.get('mainArtists', '')).strip() title = track.get('title', '').strip() common_name = f'{artist} - {title}' res_list.append({ 'album_name': track.get('albumTitle', ''), 'common_name': common_name, 'artist': artist, 'title': title, 'play_date': play_date, 'radio_name': self.radio_id, 'genre': genre }) return {'success': True, 'result': res_list, 'respond': ''}
def upload_haven(link_to_download): file_storage = new_soup(get_html(link_to_download).content) form = file_storage.find_all('form', method='POST')[0] form_all_inputs = form.find_all('input') all_inputs = { '_token': form_all_inputs[0].get('value').strip(), 'key': form_all_inputs[1].get('value').strip(), 'time': form_all_inputs[2].get('value').strip(), 'hash': form_all_inputs[2].get('value').strip() } time_sleep(6) result_form = req_post(link_to_download, headers=headers, data=all_inputs, params=all_inputs, timeout=2.50) fast_write(result_form.text, 'test.html')
def get_radios(cls): json = { 'query': '{ brand' f'(id: {cls.parent_id}) ' '{ id liveStream webRadios { id liveStream description} } } ' } headers = {'x-token': APIConfig.radiofrance_api_key} res = req_post(url=cls.tracks_request_url, json=json, headers=headers) if res.status_code != 200: return {'success': False, 'result': res.text} radio_url = cls.url radio = json_loads(res.text)['data']['brand'] radios = radio['webRadios'] fip_radio = { 'name': radio['id'], 'url': radio_url, 'stream_url': radio['liveStream'], 'description': cls.description, 'genre': cls.genre, 'country': cls.country } fip_stations = {} for radio in radios: fip_stations[radio['id']] = { 'name': radio['id'], 'url': radio_url, 'stream_url': radio['liveStream'], 'genre': radio['id'].split('_')[-1], 'description': radio['description'], 'country': cls.country } fip_stations[fip_radio['name']] = fip_radio for excluded_station in cls.exclude_radios: if excluded_station in fip_stations: del fip_stations[excluded_station] return { 'success': len(fip_stations) > 0, 'result': fip_stations, 'respond': '' }
def get_user_data(self, code_data): print('get_user_data') print(code_data) serializer = UserUAoauthSerializer(data=code_data) serializer.is_valid(raise_exception=True) data = { "access_token": serializer.validated_data['access_token'], "user_id": serializer.validated_data['user_id'] } res = req_post(USER_INFO_URL, data=data) print(USER_INFO_URL + '?=' + "access_token=" + serializer.validated_data['access_token'] + '&' "user_id=" + serializer.validated_data['user_id']) if res.status_code == 200: return res.json() elif res.status_code == 401: raise ValidationError(res.json())
def get_tracks_by_threads(one_date): start_time = str(int(one_date.timestamp())) end_time = str( int((one_date + timedelta(hours=time_step)).timestamp())) genre = self.radio_id.split('_')[-1].lower() url = self.tracks_request_url json = { 'query': '{ grid' f'(start: {start_time}, end: {end_time}, station: {self.radio_id}) ' '{ ... on TrackStep { start track { mainArtists title albumTitle } } } } ' } headers = {'x-token': APIConfig.radiofrance_api_key} res = req_post(url=url, json=json, headers=headers) responds_list.append((res.status_code, res.text)) if res.status_code != 200: return { 'success': False, 'result': 'Failed to get tracks from API', 'respond': '' } tracks = json_loads(res.text)['data']['grid'] for track in tracks: play_time = track.get('start', '') track_play_date = datetime.fromtimestamp(play_time) track = track['track'] if track: artist = ','.join(track.get('mainArtists', '')).strip() title = track.get('title', '').strip() common_name = f'{artist} - {title}' to_add = { 'album_name': track.get('albumTitle', ''), 'common_name': common_name, 'artist': artist, 'title': title, 'play_date': track_play_date, 'radio_name': self.radio_id, 'genre': genre } if to_add not in res_list: res_list.append(to_add)
def get_latest_tracks(self): """ 0 tracks - current track ∟ timetoplay - time after which need to make next request to get new current playing other tracks - previous played (9 tracks) """ latest_tracks = [] res = req_post(self.current_playing_url, data={'origin': 'website'}) res = json_loads(res.text)['tracks'] for one_track in res: seconds_to_play = one_track.get('timetoplay', 0) + 25 seconds_playing = int(one_track['duration']) - seconds_to_play start_date = datetime.now() - timedelta(seconds=seconds_playing) end_date = datetime.now() + timedelta(seconds=seconds_to_play) spotify = one_track['spotify'] if spotify: spotify = spotify.get('id', '').split('track:')[-1] deezer = one_track['deezer'] if deezer: deezer = deezer.get('id', '') track_dict = { 'artist': one_track['artist'], 'title': one_track['title'], 'common_name': f"{one_track['artist']} - {one_track['title']}", 'timetoplay': seconds_to_play, 'spotify': spotify, 'deezer': deezer, 'radio_name': self.radio_id, 'start_date': start_date, 'end_date': end_date } latest_tracks.append(track_dict) return latest_tracks
def get_acess_token(self, request): print('get_acess_token') serializer = BearesSerializer(data=request.data) serializer.is_valid(raise_exception=True) ##self.redirect_uri = serializer.validated_data['redirect_uri'], data = { "grant_type": "authorization_code", "client_id": CLIENT_ID, "client_secret": CLIENT_SECRET, "code": serializer.validated_data['code'], ##"redirect_uri": self.redirect_uri } try: res = req_post(ACCESS_TOKEN_URL, data=data) except exceptions.ConnectionError as e: raise ValidationError({ 'non_field_errors': 'Не вдалось підєднатись до серверу автоизації', 'details': str(e) }) if res.status_code == 200: return res.json() elif res.status_code == 401: raise ValidationError(res.json())
def parser(original): debug(original, 'Parse Original Message:') # message = spell_checker(original) # if message: message = [original] debug('Message After spell check: {}'.format(message)) tmpslot = dict( originalText=[], lemma=[], pos=[], slot_tag=[], ) try: slot_response = req_post('http://13.228.72.161:8080/tag', json={ "q": message }).json() general_slots = slot_response.get('result') or [('LEXICA_SLOT', ) * 4] debug('general_slots: {}'.format(general_slots)) (tmpslot['originalText'], tmpslot['lemma'], tmpslot['pos'], tmpslot['slot_tag']) = zip(*general_slots) except JSONDecodeError as err: debug('JSON error [{}]'.format(err), 'ERROR') message = ' '.join(message) predict_result = dict( zip(('label', 'original', 'stanford'), predict(message))) debug(predict_result, name='Predict Result:') # predict_result['stanford'] = predict_result.get('stanford')[0] label_score = float(predict_result.get('label')[0].get('score')) original_score = float(predict_result.get('original')[0].get('score')) if label_score > original_score: class_probability = predict_result.get('label') else: class_probability = predict_result.get('original') del label_score, original_score, predict_result words = list( filter( None, [word.strip() for word in sub(r'[^\w\s]', ' ', message).split()])) # create new words for BookSlot_MEMM prevent mutation of words book_slot = BookSlot_MEMM(words[:]) general_slots = list(map(list, zip(*list(tmpslot.values())))) book_keyword = '' for i, val in enumerate(words): if book_slot[i] == 'book': book_keyword += ' ' + val book_keyword = book_keyword.strip() if book_keyword.strip() else message debug(book_slot, name='book_slot: ') debug(book_keyword, name='book_keyword: ') debug(class_probability, name='class_probability: ') debug(general_slots, name='general_slots: ') debug(words, name='words: ') return message, book_keyword, class_probability, general_slots, words
def get_current_track(self): # orig_date = datetime.now().strftime('%d/%m/%Y %H:%M:%S') url = self.tracks_request_url json = { 'query': '{ live' f'(station: {self.radio_id}) ' '{ song { start end track { mainArtists title } } } } ' } headers = {'x-token': APIConfig.radiofrance_api_key} res = req_post(url=url, json=json, headers=headers) if res.status_code != 200: return { 'success': False, 'result': 'Failed to get tracks from API', 'respond': '' } track = json_loads(res.text)['data']['live']['song'] if not track: now = datetime.now() return { 'common_name': 'unknown - unknown', 'artist': 'unknown', 'title': 'unknown', 'start_date': now, 'end_date': now + timedelta(seconds=30), 'radio_name': self.radio_id, 'timetoplay': 30, 'spotify': '', 'deezer': '' } start_time = track.get('start', '') start_date = datetime.fromtimestamp(start_time) end_time = track.get('end', '') end_date = datetime.fromtimestamp(end_time) timetoplay = (end_date - datetime.now()).seconds track = track['track'] if track: artist = ','.join(track.get('mainArtists', '')).strip() title = track.get('title', '').strip() common_name = f'{artist} - {title}' return { 'common_name': common_name, 'artist': artist, 'title': title, 'start_date': start_date, 'end_date': end_date, 'radio_name': self.radio_id, 'timetoplay': timetoplay, 'spotify': '', 'deezer': '' } return {}
def parse(self, response): meta = response.meta item = meta['item'] username = item["username"] try: cookies_dict = dict() req_ss = req_session() index_res = req_ss.get(self._start_url_, headers=self.headers, verify=False) cookies_dict.update(index_res.cookies.get_dict()) # 获取cookies index_cookie_url = 'https://v4.passport.sohu.com/i/cookie/common' \ '?callback=passport403_cb%s&_=%s' % (get_js_time(), get_js_time()) index_cookie_res = req_ss.get(index_cookie_url, verify=False) cookies_dict.update(index_cookie_res.cookies.get_dict()) code_callback_url = 'https://v4.passport.sohu.com/i/jf/code?callback=passport403_cb%s' \ '&type=0&_=%s' % (get_js_time(), get_js_time()) code_callback_res = req_ss.get(code_callback_url, verify=False) cookies_dict.update(code_callback_res.cookies.get_dict()) # 解析 jv_val = self.js_driver(code_callback_res.text).split('=') jv_dict = {jv_val[0]: jv_val[1].split(';', 1)[0]} req_ss.cookies.update(jv_dict) cookies_dict.update(jv_dict) data = { 'userid': username, 'password': md5(item["password"].encode('utf-8')).hexdigest(), 'appid': '101305', 'callback': 'passport403_cb' + get_js_time() } login_url = 'https://v4.passport.sohu.com/i/login/101305' res = req_ss.post(login_url, data=data) result = json_loads(self.result_pattern2.search(res.text).group(1)) status = result['status'] if status == 404: yield from self.error_handle(username, "搜狐邮箱---账号密码错误", tell_msg="请刷新页面重试") return elif status == 465: callback_url = 'https://v4.passport.sohu.com/i/jf/code?callback=passport403_cb%s' \ '&type=0&_=%s' % (get_js_time(), get_js_time()) jv_content = req_ss.get(callback_url) cookies_dict.update(jv_content.cookies.get_dict()) code_callback_res = req_ss.get(code_callback_url, verify=False) jv = self.js_driver(code_callback_res.text).split('=') cookies_dict.update({jv[0]: jv[1].split(';', 1)[0]}) pagetoken = get_js_time() captcha_url = 'https://v4.passport.sohu.com/i/captcha/picture?pagetoken=%s' \ '&random=passport403_sdk%s' % (pagetoken, get_js_time()) self.set_image_captcha_headers_to_ssdb(cookies_dict, username) self.set_email_img_url_to_ssdb(captcha_url, username) captcha_body = req_get(captcha_url, cookies=cookies_dict).content captcha_code = self.ask_image_captcha(captcha_body, username, file_type=".png") data.update({ 'captcha': captcha_code, 'pagetoken': str(pagetoken) }) res = req_post(login_url, data=data, cookies=cookies_dict, verify=False) result_two = json_loads( self.result_pattern2.search(res.text).group(1)) status_two = result_two['status'] if status_two == 420: yield from self.error_handle(username, "搜狐邮箱---输入验证错误!", tell_msg="输入验证错误!") return elif status_two != 200: yield from self.error_handle(username, "搜狐邮箱---未知错误!", tell_msg="抓取失败!") return if res: # 登录成功之后需发送回滚请求, cookies_dict.update(res.cookies.get_dict()) callback_url = 'https://mail.sohu.com/fe/login/callback' call_back = req_ss.post(callback_url, verify=False) # 更新回滚 cookie cookies_dict.update(call_back.cookies.get_dict()) # 构造登出cookies logout_con = req_ss.get( 'https://v4.passport.sohu.com/i/jf/code?callback=passport403_cb%s' '&type=0&_=%s' % (get_js_time(), get_js_time()), cookies=cookies_dict, verify=False) ppmdig_cookies = logout_con.cookies.get_dict() logout_val = self.js_driver(logout_con.text).split('=') ppmdig_cookies.update( {logout_val[0]: logout_val[1].split(';', 1)[0]}) search_url = self.search_url % (0, get_js_time(), self.keyword) meta["cookies_dict"] = cookies_dict meta["ppmdig_cookies"] = ppmdig_cookies yield Request(url=search_url, cookies=cookies_dict, callback=self.parse_search, meta=meta, dont_filter=True, errback=self.err_callback) # 登录成功之后直接访问json页面,提取对应数据进行筛选 # 登录成功后,用户用可能讲账单添加到自定义文件夹里,这里需要筛选用户自定义内容,自定的新ID为17开始 # 首先判断是否存在自定义标签 else: yield from self.error_handle(username, "搜狐邮箱---账号密码错误!", tell_msg="账号或密码错误,请刷新页面重试") except Exception: yield from self.except_handle(username, "登录入口异常", tell_msg="邮箱登录失败,请刷新页面重试")
def board_display(board): global BOARDS if board == "favicon.ico": return render_template("404.html"), 404 with open(ROOT + "boardlist") as f: BOARDS = load(f) if board in BOARDS: desc = BOARDS[board] with open(ROOT + "boards/{}/index".format(board)) as f: board_content = load(f) else: return render_template("404.html"), 404 if request.method == "POST": global POSTS title = request.form["title"] if request.form.get("title", None) else "" name = "Anonymous" body = request.form["body"] if request.form.get("body", None) else "" ip = request.headers.get("X-Forwarded-For", None) logging.info(ip) if not request.form.get("g-recaptcha-response", None): flash("You'll need to do something with the captcha please.", "error") return render_template("posted.html", board=board, desc=desc) try: with open("/home/blha303/recaptchakey") as f: captcha_resp = req_post("https://www.google.com/recaptcha/api/siteverify", data={"secret": f.read().strip(), "response": request.form.get("g-recaptcha-response"), "remoteip": ip}).json() if not captcha_resp["success"]: flash("Your captcha wasn't up to par. Want to try again? (You'll have to retype your message, sorry.)", "error") return render_template("posted.html", board=board, desc=desc) except: logging.exception("Error with captcha") flash("Sorry, there was a problem while processing the captcha. Please <a href='https://twitter.com/blha303'>let me know</a>.", "error") return render_template("posted.html", board=board, desc=desc) if '<div ' in body: flash("Hi there. Sorry to rain on your parade, but I can't let you do that. Soz.", "error") return render_template("posted.html", board=board, desc=desc) if clean_html(html(body).strip()) == "<div></div>": body = "" id = request.form["id"] if request.form.get("id", None) else "" if len(body) > 1500 or len(title) > 30 or len(name) > 30: flash("Too long! Want to try that again?", "error") return render_template("posted.html", board=board, desc=desc) if not body: flash("No body provided! We kinda need something there, sorry.", "error") return render_template("posted.html", board=board, desc=desc) if not board in POSTS: POSTS[board] = 1 if id and id.isdigit(): changed_something = False for post in board_content: if post[0] == int(id): POSTS[board] += 1 post.append([int(datetime.timestamp(datetime.utcnow())), POSTS[board] + 1, body]) changed_something = True if not changed_something: flash("Sorry, couldn't find that top-level post.", "error") return render_template("posted.html", board=board, desc=desc) else: POSTS[board] += 1 board_content.append([POSTS[board] + 1, title, [int(datetime.timestamp(datetime.utcnow())), POSTS[board] + 1, body] ]) with open(ROOT + "postnums", "w") as f: dump(POSTS, f) with open(ROOT + "boards/{}/index".format(board), "w") as f: dump(board_content, f) flash("Success! (?)", "success") return render_template("posted.html", board=board, desc=desc, id=POSTS[board]) toplevel = process_board(board_content) return render_template("board.html", posts=toplevel, board=board, desc=desc)