def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
webpage_url = 'http://www.4tube.com/videos/' + video_id
webpage = self._download_webpage(webpage_url, video_id)
self.report_extraction(video_id)
playlist_json = self._html_search_regex(r'var playerConfigPlaylist\s+=\s+([^;]+)', webpage, 'Playlist')
media_id = self._search_regex(r'idMedia:\s*(\d+)', playlist_json, 'Media Id')
sources = self._search_regex(r'sources:\s*\[([^\]]*)\]', playlist_json, 'Sources').split(',')
title = self._search_regex(r'title:\s*"([^"]*)', playlist_json, 'Title')
thumbnail_url = self._search_regex(r'image:\s*"([^"]*)', playlist_json, 'Thumbnail', fatal=False)
uploader_str = self._search_regex(r'<span>Uploaded by</span>(.*?)<span>', webpage, 'uploader', fatal=False)
mobj = re.search(r'<a href="/sites/(?P<id>[^"]+)"><strong>(?P<name>[^<]+)</strong></a>', uploader_str)
(uploader, uploader_id) = (mobj.group('name'), mobj.group('id')) if mobj else (clean_html(uploader_str), None)
upload_date = None
view_count = None
duration = None
description = self._html_search_meta('description', webpage, 'description')
if description:
upload_date = self._search_regex(r'Published Date: (\d{2} [a-zA-Z]{3} \d{4})', description, 'upload date',
fatal=False)
if upload_date:
upload_date = unified_strdate(upload_date)
view_count = self._search_regex(r'Views: ([\d,\.]+)', description, 'view count', fatal=False)
if view_count:
view_count = str_to_int(view_count)
duration = parse_duration(self._search_regex(r'Length: (\d+m\d+s)', description, 'duration', fatal=False))
token_url = "http://tkn.4tube.com/{0}/desktop/{1}".format(media_id, "+".join(sources))
headers = {
b'Content-Type': b'application/x-www-form-urlencoded',
b'Origin': b'http://www.4tube.com',
}
token_req = compat_urllib_request.Request(token_url, b'{}', headers)
tokens = self._download_json(token_req, video_id)
formats = [{
'url': tokens[format]['token'],
'format_id': format + 'p',
'resolution': format + 'p',
'quality': int(format),
} for format in sources]
self._sort_formats(formats)
return {
'id': video_id,
'title': title,
'formats': formats,
'thumbnail': thumbnail_url,
'uploader': uploader,
'uploader_id': uploader_id,
'upload_date': upload_date,
'view_count': view_count,
'duration': duration,
'age_limit': 18,
'webpage_url': webpage_url,
}
def test_clean_html(self):
self.assertEqual(clean_html('a:\nb'), 'a: b')
self.assertEqual(clean_html('a:\n "b"'), 'a: "b"')
def test_clean_html_links(self):
self.assertAlmostEqual(
clean_html('<a href="http://myspace.com"> myspace </a>'),
'myspace')
def test_clean_html_advance(self):
self.assertEqual(
clean_html(
'<HTML><HEAD><TITLE>TEST <TITLE></HEAD><BODY BGCOLOR="EEEEEE"><H1>this is </H1><H2>a test </H2><BR><P><BODY>checkout myspace.com<table border="1" cellpadding="10" width="80%"><img src="img_myspace.jpg" alt="myspace.com"></BODY></HTML>'
), 'TEST this is a test checkout myspace.com')
def test_clean_html_moderate(self):
self.assertEqual(
clean_html(
'<HTML><HEAD><TITLE> this is a test </TITLE></HEAD><BR><P>here is a line of text</HTML>'
), 'this is a test here is a line of text')
def test_clean_html_basic(self):
self.assertEqual(
clean_html('<HTML><HEAD> this is a test</HEAD><HTML>'),
'this is a test')
def test_clean_html_base(self):
self.assertEqual(clean_html('This should return the exact same text'),
'This should return the exact same text')
def test_clean_html(self):
self.assertEqual(clean_html('a:\nb'), 'a: b')
self.assertEqual(clean_html('a:\n "b"'), 'a: "b"')
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
webpage_url = 'http://www.4tube.com/videos/' + video_id
webpage = self._download_webpage(webpage_url, video_id)
self.report_extraction(video_id)
playlist_json = self._html_search_regex(
r'var playerConfigPlaylist\s+=\s+([^;]+)', webpage, 'Playlist')
media_id = self._search_regex(r'idMedia:\s*(\d+)', playlist_json,
'Media Id')
sources = self._search_regex(r'sources:\s*\[([^\]]*)\]', playlist_json,
'Sources').split(',')
title = self._search_regex(r'title:\s*"([^"]*)', playlist_json,
'Title')
thumbnail_url = self._search_regex(r'image:\s*"([^"]*)',
playlist_json,
'Thumbnail',
fatal=False)
uploader_str = self._search_regex(
r'<span>Uploaded by</span>(.*?)<span>',
webpage,
'uploader',
fatal=False)
mobj = re.search(
r'<a href="/sites/(?P<id>[^"]+)"><strong>(?P<name>[^<]+)</strong></a>',
uploader_str)
(uploader, uploader_id) = (mobj.group('name'),
mobj.group('id')) if mobj else (
clean_html(uploader_str), None)
upload_date = None
view_count = None
duration = None
description = self._html_search_meta('description', webpage,
'description')
if description:
upload_date = self._search_regex(
r'Published Date: (\d{2} [a-zA-Z]{3} \d{4})',
description,
'upload date',
fatal=False)
if upload_date:
upload_date = unified_strdate(upload_date)
view_count = self._search_regex(r'Views: ([\d,\.]+)',
description,
'view count',
fatal=False)
if view_count:
view_count = str_to_int(view_count)
duration = parse_duration(
self._search_regex(r'Length: (\d+m\d+s)',
description,
'duration',
fatal=False))
token_url = "http://tkn.4tube.com/{0}/desktop/{1}".format(
media_id, "+".join(sources))
headers = {
b'Content-Type': b'application/x-www-form-urlencoded',
b'Origin': b'http://www.4tube.com',
}
token_req = compat_urllib_request.Request(token_url, b'{}', headers)
tokens = self._download_json(token_req, video_id)
formats = [{
'url': tokens[format]['token'],
'format_id': format + 'p',
'resolution': format + 'p',
'quality': int(format),
} for format in sources]
self._sort_formats(formats)
return {
'id': video_id,
'title': title,
'formats': formats,
'thumbnail': thumbnail_url,
'uploader': uploader,
'uploader_id': uploader_id,
'upload_date': upload_date,
'view_count': view_count,
'duration': duration,
'age_limit': 18,
'webpage_url': webpage_url,
}
