def get_http_data(url,
header=None,
data=None,
useragent=FIREFOX_UA,
referer=None,
cookiejar=None):
""" Get the page to parse it for streams """
if not cookiejar:
cookiejar = CookieJar()
log.debug("HTTP getting %r", url)
starttime = time.time()
request = Request(url)
standard_header = {'Referer': referer, 'User-Agent': useragent}
for key, value in [head for head in standard_header.items() if head[1]]:
request.add_header(key, value)
if header:
for key, value in [head for head in header.items() if head[1]]:
request.add_header(key, value)
if data:
request.add_data(data)
opener = build_opener(HTTPCookieProcessor(cookiejar))
try:
response = opener.open(request)
except HTTPError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.code)
sys.exit(5)
except URLError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.reason)
sys.exit(5)
except ValueError as e:
log.error("Try adding http:// before the url")
sys.exit(5)
if is_py3:
data = response.read()
try:
data = data.decode("utf-8")
except UnicodeDecodeError:
pass
else:
try:
data = response.read()
except socket.error as e:
log.error("Lost the connection to the server")
sys.exit(5)
response.close()
spent_time = time.time() - starttime
bps = 8 * len(data) / max(spent_time, 0.001)
log.debug("HTTP got %d bytes from %r in %.2fs (= %dbps)", len(data), url,
spent_time, bps)
return data
def download(self):
""" Get the stream from HTTP """
request = Request(self.url)
request.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
try:
response = urlopen(request)
except HTTPError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.code)
sys.exit(5)
try:
total_size = response.info()['Content-Length']
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 0
if self.options.output != "-":
extension = re.search(r"(\.[a-z0-9]+)$", self.url)
if extension:
self.options.output = self.options.output + extension.group(1)
else:
self.options.output = "%s.mp4" % self.options.output
log.info("Outfile: %s", self.options.output)
if os.path.isfile(self.options.output) and not self.options.force:
log.info("File already exists. use --force to overwrite")
return
file_d = open(self.options.output, "wb")
else:
file_d = sys.stdout
lastprogress = 0
while 1:
chunk = response.read(8192)
bytes_so_far += len(chunk)
if not chunk:
break
file_d.write(chunk)
if self.options.output != "-":
now = time.time()
if lastprogress + 1 < now:
lastprogress = now
progress(bytes_so_far, total_size)
if self.options.output != "-":
file_d.close()
def download(self):
""" Get the stream from HTTP """
request = Request(self.url)
request.add_header(
'User-Agent',
'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3'
)
try:
response = urlopen(request)
except HTTPError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.code)
sys.exit(5)
try:
total_size = response.info()['Content-Length']
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 0
file_d = output(self.options, self.options.output, "mp4")
if hasattr(file_d, "read") is False:
return
lastprogress = 0
while 1:
chunk = response.read(8192)
bytes_so_far += len(chunk)
if not chunk:
break
file_d.write(chunk)
if self.options.output != "-":
now = time.time()
if lastprogress + 1 < now:
lastprogress = now
progress(bytes_so_far, total_size)
if self.options.output != "-":
file_d.close()
def get_http_data(url, header=None, data=None, useragent=FIREFOX_UA,
referer=None, cookiejar=None):
""" Get the page to parse it for streams """
if not cookiejar:
cookiejar = CookieJar()
log.debug("HTTP getting %r", url)
starttime = time.time()
request = Request(url)
standard_header = {'Referer': referer, 'User-Agent': useragent}
for key, value in [head for head in standard_header.items() if head[1]]:
request.add_header(key, value)
if header:
for key, value in [head for head in header.items() if head[1]]:
request.add_header(key, value)
if data:
request.add_data(data)
opener = build_opener(HTTPCookieProcessor(cookiejar))
try:
response = opener.open(request)
except HTTPError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.code)
sys.exit(5)
except URLError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.reason)
sys.exit(5)
except ValueError as e:
log.error("Try adding http:// before the url")
sys.exit(5)
if is_py3:
data = response.read()
try:
data = data.decode("utf-8")
except UnicodeDecodeError:
pass
else:
try:
data = response.read()
except socket.error as e:
log.error("Lost the connection to the server")
sys.exit(5)
response.close()
spent_time = time.time() - starttime
bps = 8 * len(data) / max(spent_time, 0.001)
log.debug("HTTP got %d bytes from %r in %.2fs (= %dbps)",
len(data), url, spent_time, bps)
return data
def download(self):
""" Get the stream from HTTP """
request = Request(self.url)
request.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
try:
response = urlopen(request)
except HTTPError as e:
log.error("Something wrong with that url")
log.error("Error code: %s", e.code)
sys.exit(5)
try:
total_size = response.info()['Content-Length']
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 0
file_d = output(self.options, self.url, "mp4")
if hasattr(file_d, "read") is False:
return
lastprogress = 0
while 1:
chunk = response.read(8192)
bytes_so_far += len(chunk)
if not chunk:
break
file_d.write(chunk)
if self.options.output != "-":
now = time.time()
if lastprogress + 1 < now:
lastprogress = now
progress(bytes_so_far, total_size)
if self.options.output != "-":
file_d.close()
def get_http_data(url, header=None, post=None, useragent=FIREFOX_UA,
referer=None, cookiejar=None):
""" Get the page to parse it for streams """
if not cookiejar:
cookiejar = CookieJar()
if url.find("manifest.f4m") > 0:
parse = urlparse(url)
url = "%s://%s%s?%s&hdcore=3.3.0" % (parse.scheme, parse.netloc, parse.path, parse.query)
log.debug("HTTP getting %r", url)
starttime = time.time()
error = None
if post:
if is_py3:
post = bytes(post, encoding="utf-8")
request = Request(url, data=post)
else:
request = Request(url)
standard_header = {'Referer': referer, 'User-Agent': useragent}
for key, value in [head for head in standard_header.items() if head[1]]:
request.add_header(key, value)
if header:
for key, value in [head for head in header.items() if head[1]]:
request.add_header(key, value)
opener = build_opener(HTTPCookieProcessor(cookiejar))
try:
response = opener.open(request)
except HTTPError as e:
error = True
data = e.read()
return error, data
if is_py3:
data = response.read()
try:
data = data.decode("utf-8")
except UnicodeDecodeError:
pass
else:
try:
data = response.read()
except socket.error as e:
return True, "Lost the connection to the server"
response.close()
spent_time = time.time() - starttime
bps = 8 * len(data) / max(spent_time, 0.001)
log.debug("HTTP got %d bytes from %r in %.2fs (= %dbps)",
len(data), url, spent_time, bps)
return error, data
