async def new(request): """ Вставка в базу новой записи. :param request: :return: """ data = await request.json() url = url_fetch(data) date = time.strftime("%Y-%m-%d") exp_date = datetime.today() + relativedelta(months=6) exp_date = exp_date.strftime("%Y-%m-%d") async with request.app['db'].acquire() as conn: cursor = await conn.execute( "INSERT INTO url (url_long, url_short, pub_date, exp_date) VALUES (%s, %s, %s, %s) RETURNING id;", url, "", date, exp_date) id = await cursor.fetchall() url_short = encode(id[0][0]) await conn.execute("UPDATE url SET url_short = %s WHERE id= %s", url_short, id[0][0]) url = "http://{host}:{port}/{path}".format( host=request.app['config']['host'], port=request.app['config']['port'], path=url_short) return web.Response(text=url)
def _fetch_url(self, url, post_data=None, parameters=None, http_method='GET'): extra_params = {} if parameters is not None: extra_params.update(parameters) if post_data: http_method = "POST" if self._oauth_consumer is not None: if post_data: parameters = post_data.copy() req = oauth.Request.from_consumer_and_token(self._oauth_consumer, token=self._oauth_token, http_method=http_method, http_url=url, parameters=parameters) req.sign_request(self._signature_method_hmac_sha1, self._oauth_consumer, self._oauth_token) if http_method == "POST": encoded_post_data = req.to_postdata() else: encoded_post_data = None url = req.to_url() else: url = self._build_url(url, extra_params=extra_params) encoded_post_data = self._encode_post_data(post_data) try: response = url_fetch(url, payload=encoded_post_data, method=http_method) except urlfetch_errors.Error: return None else: try: json = simplejson.loads(response) except ValueError: raise TwitterInternalServerError('Internal Server Error') except BaseException, e: raise TwitterError(e.message) self._check_for_twitter_error(json) return json
def dummy(): template = 'http://www.flvcd.com/parse.php?&format=super&kw=%s' content = yield utils.url_fetch(template % urllib.parse.quote_plus('http://v.youku.com/v_show/id_XNDExNTg1NTcy.html')) soup = BeautifulSoup(content.body) data = yield Youku.get_video_download_url(soup) print(data) data = yield Youku.get_video_name(soup) print(data)
def get_video_urls_from_playlist_show(playlist_id): template = 'http://www.youku.com/playlist_show/id_%s_ascending_1_mode_pic_page_%s.html' start = 1 videos = [] while True: content = yield utils.url_fetch(template % (playlist_id, start)) soup = BeautifulSoup(content.body) videos_tmp = [x['href'] for x in soup.find_all(target='video')[1:]] if not videos_tmp: break start += 1 videos += videos_tmp raise tornado.gen.Return(videos)
def get_video_urls_from_show_id(show_id): template = 'http://www.youku.com/show_episode/id_%s.html?dt=json&divid=reload_%s' start = 1 videos = [] while True: content = yield utils.url_fetch(template % (show_id, start)) soup = BeautifulSoup(content.body) videos_tmp = [x['href'] for x in soup.find_all('a')] if len(videos_tmp) == 0: break start += len(videos_tmp) videos += videos_tmp raise tornado.gen.Return(videos)
def get_video_name(video_or_soup): try: if isinstance(video_or_soup, BeautifulSoup): soup = video_or_soup else: template = 'http://www.flvcd.com/parse.php?&format=super&kw=%s' content = yield utils.url_fetch(template % urllib.parse.quote_plus(video_or_soup)) soup = BeautifulSoup(content.body) raise tornado.gen.Return(list(soup.find_all(class_='mn STYLE4')[1].children) [2].strip()) except (KeyboardInterrupt, tornado.gen.Return): raise except: tornado.log.app_log.error(traceback.format_exc())
def login_oauth(web_url, username, password): result = url_fetch(web_url) authenticity_token = '' oauth_token = '' groups = re.search(r'<input .*name="authenticity_token" .*value="(.*)"', result, re.IGNORECASE) if groups: authenticity_token = groups.group(1) groups = re.search(r'<input .*name="oauth_token" .*value="(.*)"', result, re.IGNORECASE) if groups: oauth_token = groups.group(1) if not authenticity_token or not oauth_token: raise urlfetch_errors.Error str = urllib.urlencode({'authenticity_token': authenticity_token, 'oauth_token': oauth_token, 'session[username_or_email]': username, 'session[password]': password}) result = url_fetch(SIGNIN_URL, str, 'POST') if 'Invalid user name or password' in result: raise TwitterAuthenticationError groups = re.search(r'<code>(.*)</code>', result, re.IGNORECASE) if groups: oauth_pin = groups.group(1) return oauth_pin else: raise urlfetch_errors.Error
def get_video_download_url(video_or_soup): try: if isinstance(video_or_soup, BeautifulSoup): soup = video_or_soup else: template = 'http://www.flvcd.com/parse.php?&format=super&kw=%s' content = yield utils.url_fetch(template % urllib.parse.quote_plus(video_or_soup)) soup = BeautifulSoup(content.body) urls = [x['href'] for x in soup.find_all('a', href=re.compile('getFlvPath'))] raise tornado.gen.Return(urls) except (KeyboardInterrupt, tornado.gen.Return): raise except: tornado.log.app_log.error(traceback.format_exc()) raise tornado.gen.Return([])
def get_video_name_and_download_urls(cls, video_or_soup): try: if isinstance(video_or_soup, BeautifulSoup): soup = video_or_soup else: template = 'http://www.flvcd.com/parse.php?&format=super&kw=%s' content = yield utils.url_fetch(template % urllib.parse.quote_plus(video_or_soup)) soup = BeautifulSoup(content.body) name = yield cls.get_video_name(soup) download_urls = yield cls.get_video_download_url(soup) raise tornado.gen.Return((name, download_urls)) except (KeyboardInterrupt, tornado.gen.Return): raise except: tornado.log.app_log.error(traceback.format_exc())