def get_http_data(url,
header=None,
data=None,
useragent=FIREFOX_UA,
referer=None,
cookiejar=None):
""" Get the page to parse it for streams """
if not cookiejar:
cookiejar = CookieJar()
log.debug("HTTP getting %r", url)
starttime = time.time()
request = Request(url)
standard_header = {'Referer': referer, 'User-Agent': useragent}
for key, value in [head for head in standard_header.items() if head[1]]:
request.add_header(key, value)
if header:
for key, value in [head for head in header.items() if head[1]]:
request.add_header(key, value)
if data:
request.add_data(data)
opener = build_opener(HTTPCookieProcessor(cookiejar))
try:
response = opener.open(request)
except HTTPError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.code)
sys.exit(5)
except URLError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.reason)
sys.exit(5)
except ValueError as e:
log.error("Try adding http:// before the url")
sys.exit(5)
if is_py3:
data = response.read()
try:
data = data.decode("utf-8")
except UnicodeDecodeError:
pass
else:
try:
data = response.read()
except socket.error as e:
log.error("Lost the connection to the server")
sys.exit(5)
response.close()
spent_time = time.time() - starttime
bps = 8 * len(data) / max(spent_time, 0.001)
log.debug("HTTP got %d bytes from %r in %.2fs (= %dbps)", len(data), url,
spent_time, bps)
return data
def download(self):
""" Get the stream from HTTP """
request = Request(self.url)
request.add_header(
'User-Agent',
'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3'
)
try:
response = urlopen(request)
except HTTPError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.code)
sys.exit(5)
try:
total_size = response.info()['Content-Length']
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 0
file_d = output(self.options, self.options.output, "mp4")
if hasattr(file_d, "read") is False:
return
lastprogress = 0
while 1:
chunk = response.read(8192)
bytes_so_far += len(chunk)
if not chunk:
break
file_d.write(chunk)
if self.options.output != "-":
now = time.time()
if lastprogress + 1 < now:
lastprogress = now
progress(bytes_so_far, total_size)
if self.options.output != "-":
file_d.close()
