def download_pdf(url, category):
# add domain
url = "https://www.geekbooks.me" + url
# if category directory is not exist, create
if os.path.isdir(conf_books_dir + category):
pass
else:
os.makedirs(conf_books_dir + category)
# create instance of MozillaCookieJar
cookie = cookielib.MozillaCookieJar()
# get cookie from file
cookie.load('../data/cookie4geek.data',
ignore_discard=True,
ignore_expires=True)
handler = urllib2.HTTPCookieProcessor(cookie)
opener = urllib2.build_opener(handler)
# add header
opener.addheaders = [('User-agent', 'Mozilla/5.0'), ("Referer", url)]
for url in list_pdf_url_by_book_detail_url(url):
logger.info("PageDetailPage> {url}".format(
url=("https://www.geekbooks.me" + url)))
file_name = url.split('/')[-1]
u = opener.open("https://www.geekbooks.me" + url)
print "Preparing to download..."
# f with directory
if os.path.exists(conf_books_dir + category + "/" +
file_name) and detect_book(
(conf_books_dir + category + "/" + file_name)):
continue
f = open(conf_books_dir + category + "/" + file_name, 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s Bytes: %s" % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl,
file_size_dl * 100. / file_size)
status = status + chr(8) * (len(status) + 1)
print status,
f.close()
def download_pdf(url, category):
# add domain
url = "https://www.geekbooks.me" + url
# if category directory is not exist, create
if os.path.isdir(conf_books_dir + category):
pass
else:
os.makedirs(conf_books_dir + category)
# create instance of MozillaCookieJar
cookie = cookielib.MozillaCookieJar()
# get cookie from file
cookie.load('../data/cookie4geek.data', ignore_discard=True, ignore_expires=True)
handler = urllib2.HTTPCookieProcessor(cookie)
opener = urllib2.build_opener(handler)
# add header
opener.addheaders = [('User-agent', 'Mozilla/5.0'), ("Referer", url)]
for url in list_pdf_url_by_book_detail_url(url):
logger.info("PageDetailPage> {url}".format(url=("https://www.geekbooks.me" + url)))
file_name = url.split('/')[-1]
u = opener.open("https://www.geekbooks.me" + url)
print "Preparing to download..."
# f with directory
if os.path.exists(conf_books_dir + category + "/" + file_name) and detect_book(
(conf_books_dir + category + "/" + file_name)):
continue
f = open(conf_books_dir + category + "/" + file_name, 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s Bytes: %s" % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8) * (len(status) + 1)
print status,
f.close()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from GeekBook.util.log_util import logger
if __name__ == '__main__':
logger.info("info")
logger.debug("debug")
logger.warn("warn")
logger.error("error")
pass