def do_CONNECT(self):
log.info("CONNECT %s", self.path)
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
if CONF['httpsProxy']:
if self._connect_to_proxy(CONF['httpsProxy'], self.path, soc):
self._read_write(soc, 300)
else:
if self._connect_to(self.path, soc):
self.send_response(200, "OK")
self.end_headers()
self._read_write(soc, 300)
except Exception as e:
log.error("Connection dropped, reason: %s", str(e))
finally:
soc.close()
self.connection.close()
def _file_read_write(self, fd, start, end):
offset = start
content_length = end - start + 1
if CONF['showSpeed']:
tm_a = [time.time(), offset - start]
tm_b = [time.time(), offset - start]
fd.seek(offset)
try:
while content_length > 0:
if sendfile:
sent = min(content_length, SENDFILE_MAXSIZE)
log.debug("sendfile: offset / size: %d / %d bytes", offset, sent)
sent = sendfile(self.connection.fileno(), fd.fileno(), offset, sent)
if sent <= 0:
break
else:
data = fd.read(min(content_length, CONF['bufSize']))
if not data:
break
self.connection.send(data)
sent = len(data)
offset += sent
content_length -= sent
if CONF['showSpeed']:
tm_b = [time.time(), offset - start]
delta = tm_b[0] - tm_a[0]
rest = end - offset + 1
if delta >= CONF['updateInterval'] or rest == 0:
speed = (tm_b[1] - tm_a[1]) / delta
log.info("SPD: %.2fKB/S, XFER / TODO: %d / %d bytes, ETA %d sec"
% (speed / 1000, offset - start, rest, rest / speed))
tm_a = tm_b
except (OSError, socket.error) as e:
log.error("Connection dropped, %d bytes sent, reason: %s", offset - start, str(e))
def start_server():
server_address = ("0.0.0.0", CONF['port'])
proxy_handler.ProxyHandler.protocol_version = "HTTP/1.1"
run_event = threading.Event()
httpd = ThreadingHTTPServer(server_address, proxy_handler.ProxyHandler, CONF['cache'])
sa = httpd.socket.getsockname()
log.info("Download directory: %s", CONF['downloadDIR'])
log.info("Servering HTTP on %s %s %s", sa[0], "port", sa[1])
req_count = 0
while not run_event.isSet():
try:
httpd.handle_request()
req_count += 1
if req_count == 1000:
log.info("Number of active threads: %s", threading.activeCount())
req_count = 0
except select.error, e:
if e[0] == 4 and run_event.isSet(): pass
else:
log.error("Errno: %d - %s", e[0], e[1])
except socket.error as e:
if e.errno == 10054:
log.error("Connection reset by peer")
else:
log.error(str(e))
def load_cache_list(self, fn):
if not os.path.exists(fn):
with open(fn, "w") as f:
f.write("# format in cache.txt:\n\n")
f.write("# replace_url->replace_file_path\n")
f.write("# search:replace_filename->replace_file_path\n")
f.write("# re:replace_url_regular_expression->replace_file_path\n\n")
f.write("# if replace_file_path is relative then it will be lookup from download directory\n")
try:
with open(fn, "r") as f:
for l in f:
l = l.decode(errors='ignore').strip()
if not l or l[0] == '#':
continue
if '->' in l:
url, fn = l.split('->')
else:
url = l
filename = url.split("?")[0].split("/")[-1]
if not fn:
continue
" fn如为相对路径名应加上下载目录路径 "
if not os.path.isabs(fn):
fn = os.path.join(CONF['downloadDIR'], fn)
try:
open(fn).close()
self.replace_dict[url] = fn
except IOError as e:
log.error("%s: %s", fn, str(e))
log.info("%d local caches loaded" % (len(self.replace_dict)))
log.debug("Dumping cache list:")
log.debug(self.replace_dict)
except IOError as e:
log.error(str(e))
def do_GET(self, head_only=False):
self.close_connection = 1
self.fixPSVBrokenPath()
log.info("GET %s", self.path)
(scm, netloc, path, params, query, fragment) = urlparse.urlparse(self.path, 'http')
if not path:
path = '/'
if scm not in ('http', 'ftp') or fragment or not netloc:
self.send_error(400, "bad url %s" % self.path)
return
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
if self.tryDownloadPath(self.path, head_only):
return
for url, filename in self.server.replace_dict.iteritems():
if url.startswith('re:'):
r = re.compile(url[len('re:'):])
if r.match(self.path):
replace_url = self.path
self.getLocalCache(filename, head_only)
return
if url.startswith('search:') and url[len('search:'):] in self.path:
self.getLocalCache(filename, head_only)
return
if url == self.path:
self.getLocalCache(filename, head_only)
return
if scm == 'http':
if self._connect_to(netloc, soc):
soc.send("%s %s %s\r\n" % (self.command,
urlparse.urlunparse(('', '', path,
params, query,
'')),
self.request_version))
self.headers['Connection'] = 'close'
del self.headers['Proxy-Connection']
for key_val in self.headers.items():
soc.send("%s: %s\r\n" % key_val)
soc.send("\r\n")
self._read_write(soc)
elif scm == 'ftp':
# fish out user and password information
i = netloc.find ('@')
if i >= 0:
login_info, netloc = netloc[:i], netloc[i+1:]
try: user, passwd = login_info.split (':', 1)
except ValueError: user, passwd = "anonymous", None
else: user, passwd ="anonymous", None
ftp = ftplib.FTP (netloc)
ftp.login (user, passwd)
if self.command == "GET":
ftp.retrbinary ("RETR %s"%path, self.connection.send)
ftp.quit ()
except Exception as e:
log.error("Connection dropped, reason: %s", str(e))
finally:
soc.close()
self.connection.close()