def getLocalCache(self, replace_fn, head_only):
log.info("cache redirected from %s to %s", self.path, replace_fn)
log.debug("Dumping client headers:")
for h in self.headers:
log.debug(" %s: %s", h, self.headers[h])
try:
file_length = self.getFileLength(replace_fn)
log.debug("file length: %d bytes", file_length)
start, end = 0, file_length - 1
lastModString = os.path.getmtime(replace_fn)
lastModString = datetime.datetime.utcfromtimestamp(lastModString)
lastModString = lastModString.strftime('%a, %d %b %Y %H:%M:%S GMT')
dateString = os.path.getctime(replace_fn)
dateString = datetime.datetime.utcfromtimestamp(dateString)
dateString = dateString.strftime('%a, %d %b %Y %H:%M:%S GMT')
except IOError as e:
self.send_error(500, "Internal Server Error")
return
if 'Range' in self.headers:
try:
" 在http协议中，start和end为[start, end] 范围从0开始到文件大小-1 "
start, end = self.getFileRange(replace_fn, self.headers['Range'], file_length)
except RangeError as e:
""" rfc2616: A server sending a response with status code 416
(Requested range not satisfiable) SHOULD include a Content-Range
field with a byte-range- resp-spec of "*".
The instance-length specifies the current length of the selected resource. """
self.send_response(416, 'Requested Range Not Satisfiable')
self.send_header("Content-Range", "bytes */%d" % (file_length))
return
self.send_response(206, "Partial Content")
self.send_header("Content-Range", "bytes %d-%d/%d" % (start, end, file_length))
else:
self.send_response(200, "OK")
content_length = end - start + 1
self.send_header("Accept-Ranges", "bytes")
self.send_header("Content-Type", "application/octet-stream")
self.send_header("Content-Length", "%d" % (content_length))
self.send_header("Connection", "close")
self.send_header("Last-Modified", lastModString)
self.send_header("Date", dateString)
self.end_headers()
if head_only:
return
log.debug("Range: from %u to %d", start, end)
with open(replace_fn, "rb") as fd:
if fallocate:
log.debug("posix_fadvise: start from %d bytes, %d bytes length", start, content_length)
fallocate.posix_fadvise(fd, start, content_length, fallocate.POSIX_FADV_SEQUENTIAL | fallocate.POSIX_FADV_WILLNEED)
self._file_read_write(fd, start, end)
def fixPSVBrokenPath(self):
if not CONF['fixVitaPath']:
return
last_http = self.path.rfind("http://")
if last_http != 0 and last_http != -1:
fixed = self.path[last_http:]
log.info("Fixed psvita broken path %s to %s", self.path, fixed)
self.path = fixed
def main():
try:
config.load_configure()
except:
config.save_configure()
parse_arguments(sys.argv)
log.init_logger()
log.debug("Dumping configure")
log.debug(CONF)
log.info("Any clients will be served...")
proxy_server.start_server()
def do_CONNECT(self):
log.info("CONNECT %s", self.path)
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
if CONF['httpsProxy']:
if self._connect_to_proxy(CONF['httpsProxy'], self.path, soc):
self._read_write(soc, 300)
else:
if self._connect_to(self.path, soc):
self.send_response(200, "OK")
self.end_headers()
self._read_write(soc, 300)
finally:
soc.close()
self.connection.close()
def _file_read_write(self, fd, start, end):
offset = start
content_length = end - start + 1
if CONF['showSpeed']:
tm_a = [time.time(), offset - start]
tm_b = [time.time(), offset - start]
fd.seek(offset)
try:
while content_length > 0:
if sendfile:
sent = min(content_length, SENDFILE_MAXSIZE)
log.debug("sendfile: offset / size: %d / %d bytes", offset, sent)
sent = sendfile(self.connection.fileno(), fd.fileno(), offset, sent)
if sent <= 0:
break
else:
data = fd.read(min(content_length, CONF['bufSize']))
if not data:
break
self.connection.send(data)
sent = len(data)
offset += sent
content_length -= sent
if CONF['showSpeed']:
tm_b = [time.time(), offset - start]
delta = tm_b[0] - tm_a[0]
rest = end - offset + 1
if delta >= CONF['updateInterval'] or rest == 0:
speed = (tm_b[1] - tm_a[1]) / delta
log.info("SPD: %.2fKB/S, XFER / TODO: %d / %d bytes, ETA %d sec"
% (speed / 1000, offset - start, rest, rest / speed))
tm_a = tm_b
except (OSError, socket.error) as e:
log.error("Connection dropped, %d bytes sent, reason: %s", offset - start, str(e))
def start_server():
server_address = ("0.0.0.0", CONF['port'])
proxy_handler.ProxyHandler.protocol_version = "HTTP/1.1"
run_event = threading.Event()
httpd = ThreadingHTTPServer(server_address, proxy_handler.ProxyHandler, CONF['cache'])
sa = httpd.socket.getsockname()
log.info("Download directory: %s", CONF['downloadDIR'])
log.info("Servering HTTP on %s %s %s", sa[0], "port", sa[1])
req_count = 0
while not run_event.isSet():
try:
httpd.handle_request()
req_count += 1
if req_count == 1000:
log.info("Number of active threads: %s", threading.activeCount())
req_count = 0
except select.error, e:
if e[0] == 4 and run_event.isSet(): pass
else:
log.error("Errno: %d - %s", e[0], e[1])
except socket.error as e:
if e.errno == 10054:
log.error("Connection reset by peer")
else:
log.error(str(e))
def load_cache_list(self, fn):
if not os.path.exists(fn):
with open(fn, "w") as f:
f.write("# format in cache.txt:\n\n")
f.write("# replace_url->replace_file_path\n")
f.write("# search:replace_filename->replace_file_path\n")
f.write("# re:replace_url_regular_expression->replace_file_path\n\n")
f.write("# if replace_file_path is relative then it will be lookup from download directory\n")
try:
with open(fn, "r") as f:
for l in f:
l = l.decode(errors='ignore').strip()
if not l or l[0] == '#':
continue
if '->' in l:
url, fn = l.split('->')
else:
url = l
filename = url.split("?")[0].split("/")[-1]
if not fn:
continue
" fn如为相对路径名应加上下载目录路径 "
if not os.path.isabs(fn):
fn = os.path.join(CONF['downloadDIR'], fn)
try:
open(fn).close()
self.replace_dict[url] = fn
except IOError as e:
log.error("%s: %s", fn, str(e))
log.info("%d local caches loaded" % (len(self.replace_dict)))
log.debug("Dumping cache list:")
log.debug(self.replace_dict)
except IOError as e:
log.error(str(e))
def log_message(self, format, *args):
log.info(format, *args)
def do_GET(self, head_only=False):
self.close_connection = 1
self.fixPSVBrokenPath()
log.info("GET %s", self.path)
(scm, netloc, path, params, query, fragment) = urlparse.urlparse(self.path, 'http')
if not path:
path = '/'
if scm not in ('http', 'ftp') or fragment or not netloc:
self.send_error(400, "bad url %s" % self.path)
return
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
if self.tryDownloadPath(self.path, head_only):
return
for url, filename in self.server.replace_dict.iteritems():
if url.startswith('re:'):
r = re.compile(url[len('re:'):])
if r.match(self.path):
replace_url = self.path
self.getLocalCache(filename, head_only)
return
if url.startswith('search:') and url[len('search:'):] in self.path:
self.getLocalCache(filename, head_only)
return
if url == self.path:
self.getLocalCache(filename, head_only)
return
if scm == 'http':
if self._connect_to(netloc, soc):
soc.send("%s %s %s\r\n" % (self.command,
urlparse.urlunparse(('', '', path,
params, query,
'')),
self.request_version))
self.headers['Connection'] = 'close'
del self.headers['Proxy-Connection']
for key_val in self.headers.items():
soc.send("%s: %s\r\n" % key_val)
soc.send("\r\n")
self._read_write(soc)
elif scm == 'ftp':
# fish out user and password information
i = netloc.find ('@')
if i >= 0:
login_info, netloc = netloc[:i], netloc[i+1:]
try: user, passwd = login_info.split (':', 1)
except ValueError: user, passwd = "anonymous", None
else: user, passwd ="anonymous", None
ftp = ftplib.FTP (netloc)
ftp.login (user, passwd)
if self.command == "GET":
ftp.retrbinary ("RETR %s"%path, self.connection.send)
ftp.quit ()
except Exception as e:
log.error("Connection dropped, reason: %s", str(e))
finally:
soc.close()
self.connection.close()
def start_server():
server_address = ("0.0.0.0", CONF['port'])
proxy_handler.ProxyHandler.protocol_version = "HTTP/1.1"
run_event = threading.Event()
httpd = ThreadingHTTPServer(server_address, proxy_handler.ProxyHandler, CONF['cache'])
sa = httpd.socket.getsockname()
log.info("Download directory: %s", CONF['downloadDIR'])
log.info("Servering HTTP on %s %s %s", sa[0], "port", sa[1])
req_count = 0
while not run_event.isSet():
try:
httpd.handle_request()
req_count += 1
if req_count == 1000:
log.info("Number of active threads: %s", threading.activeCount())
req_count = 0
except select.error, e:
if e[0] == 4 and run_event.isSet(): pass
else:
log.error("Errno: %d - %s", e[0], e[1])
except socket.error as e:
if e.errno == 10054:
log.error("Connection reset by peer")
else:
log.error(str(e))
log.info("Server shutdown")
# vim: set tabstop=4 sw=4 expandtab:
