async def new(request):
"""
Вставка в базу новой записи.
:param request:
:return:
"""
data = await request.json()
url = url_fetch(data)
date = time.strftime("%Y-%m-%d")
exp_date = datetime.today() + relativedelta(months=6)
exp_date = exp_date.strftime("%Y-%m-%d")
async with request.app['db'].acquire() as conn:
cursor = await conn.execute(
"INSERT INTO url (url_long, url_short, pub_date, exp_date) VALUES (%s, %s, %s, %s) RETURNING id;",
url, "", date, exp_date)
id = await cursor.fetchall()
url_short = encode(id[0][0])
await conn.execute("UPDATE url SET url_short = %s WHERE id= %s",
url_short, id[0][0])
url = "http://{host}:{port}/{path}".format(
host=request.app['config']['host'],
port=request.app['config']['port'],
path=url_short)
return web.Response(text=url)
def _fetch_url(self, url, post_data=None, parameters=None, http_method='GET'):
extra_params = {}
if parameters is not None:
extra_params.update(parameters)
if post_data:
http_method = "POST"
if self._oauth_consumer is not None:
if post_data:
parameters = post_data.copy()
req = oauth.Request.from_consumer_and_token(self._oauth_consumer, token=self._oauth_token,
http_method=http_method, http_url=url,
parameters=parameters)
req.sign_request(self._signature_method_hmac_sha1, self._oauth_consumer, self._oauth_token)
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
else:
url = self._build_url(url, extra_params=extra_params)
encoded_post_data = self._encode_post_data(post_data)
try:
response = url_fetch(url, payload=encoded_post_data, method=http_method)
except urlfetch_errors.Error:
return None
else:
try:
json = simplejson.loads(response)
except ValueError:
raise TwitterInternalServerError('Internal Server Error')
except BaseException, e:
raise TwitterError(e.message)
self._check_for_twitter_error(json)
return json
def dummy():
template = 'http://www.flvcd.com/parse.php?&format=super&kw=%s'
content = yield utils.url_fetch(template %
urllib.parse.quote_plus('http://v.youku.com/v_show/id_XNDExNTg1NTcy.html'))
soup = BeautifulSoup(content.body)
data = yield Youku.get_video_download_url(soup)
print(data)
data = yield Youku.get_video_name(soup)
print(data)
def get_video_urls_from_playlist_show(playlist_id):
template = 'http://www.youku.com/playlist_show/id_%s_ascending_1_mode_pic_page_%s.html'
start = 1
videos = []
while True:
content = yield utils.url_fetch(template % (playlist_id, start))
soup = BeautifulSoup(content.body)
videos_tmp = [x['href'] for x in soup.find_all(target='video')[1:]]
if not videos_tmp:
break
start += 1
videos += videos_tmp
raise tornado.gen.Return(videos)
def get_video_urls_from_show_id(show_id):
template = 'http://www.youku.com/show_episode/id_%s.html?dt=json&divid=reload_%s'
start = 1
videos = []
while True:
content = yield utils.url_fetch(template % (show_id, start))
soup = BeautifulSoup(content.body)
videos_tmp = [x['href'] for x in soup.find_all('a')]
if len(videos_tmp) == 0:
break
start += len(videos_tmp)
videos += videos_tmp
raise tornado.gen.Return(videos)
def get_video_name(video_or_soup):
try:
if isinstance(video_or_soup, BeautifulSoup):
soup = video_or_soup
else:
template = 'http://www.flvcd.com/parse.php?&format=super&kw=%s'
content = yield utils.url_fetch(template % urllib.parse.quote_plus(video_or_soup))
soup = BeautifulSoup(content.body)
raise tornado.gen.Return(list(soup.find_all(class_='mn STYLE4')[1].children)
[2].strip())
except (KeyboardInterrupt, tornado.gen.Return):
raise
except:
tornado.log.app_log.error(traceback.format_exc())
def login_oauth(web_url, username, password):
result = url_fetch(web_url)
authenticity_token = ''
oauth_token = ''
groups = re.search(r'<input .*name="authenticity_token" .*value="(.*)"', result, re.IGNORECASE)
if groups:
authenticity_token = groups.group(1)
groups = re.search(r'<input .*name="oauth_token" .*value="(.*)"', result, re.IGNORECASE)
if groups:
oauth_token = groups.group(1)
if not authenticity_token or not oauth_token:
raise urlfetch_errors.Error
str = urllib.urlencode({'authenticity_token': authenticity_token, 'oauth_token': oauth_token,
'session[username_or_email]': username, 'session[password]': password})
result = url_fetch(SIGNIN_URL, str, 'POST')
if 'Invalid user name or password' in result:
raise TwitterAuthenticationError
groups = re.search(r'<code>(.*)</code>', result, re.IGNORECASE)
if groups:
oauth_pin = groups.group(1)
return oauth_pin
else:
raise urlfetch_errors.Error
def get_video_download_url(video_or_soup):
try:
if isinstance(video_or_soup, BeautifulSoup):
soup = video_or_soup
else:
template = 'http://www.flvcd.com/parse.php?&format=super&kw=%s'
content = yield utils.url_fetch(template % urllib.parse.quote_plus(video_or_soup))
soup = BeautifulSoup(content.body)
urls = [x['href'] for x in soup.find_all('a', href=re.compile('getFlvPath'))]
raise tornado.gen.Return(urls)
except (KeyboardInterrupt, tornado.gen.Return):
raise
except:
tornado.log.app_log.error(traceback.format_exc())
raise tornado.gen.Return([])
def get_video_name_and_download_urls(cls, video_or_soup):
try:
if isinstance(video_or_soup, BeautifulSoup):
soup = video_or_soup
else:
template = 'http://www.flvcd.com/parse.php?&format=super&kw=%s'
content = yield utils.url_fetch(template % urllib.parse.quote_plus(video_or_soup))
soup = BeautifulSoup(content.body)
name = yield cls.get_video_name(soup)
download_urls = yield cls.get_video_download_url(soup)
raise tornado.gen.Return((name, download_urls))
except (KeyboardInterrupt, tornado.gen.Return):
raise
except:
tornado.log.app_log.error(traceback.format_exc())
