Exemple #1
0
    def do_CONNECT(self):
        log.info("CONNECT %s", self.path)
        soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        try:
            if CONF['httpsProxy']:
                if self._connect_to_proxy(CONF['httpsProxy'], self.path, soc):
                    self._read_write(soc, 300)
            else:
                if self._connect_to(self.path, soc):
                    self.send_response(200, "OK")
                    self.end_headers()
                    self._read_write(soc, 300)
        except Exception as e:
            log.error("Connection dropped, reason: %s", str(e))
        finally:
            soc.close()
            self.connection.close()
Exemple #2
0
    def _file_read_write(self, fd, start, end):
        offset = start
        content_length = end - start + 1

        if CONF['showSpeed']:
            tm_a = [time.time(), offset - start]
            tm_b = [time.time(), offset - start]

        fd.seek(offset)
        try:
            while content_length > 0:
                if sendfile:
                    sent = min(content_length, SENDFILE_MAXSIZE)
                    log.debug("sendfile: offset / size: %d / %d bytes", offset, sent)
                    sent = sendfile(self.connection.fileno(), fd.fileno(), offset, sent)
                    if sent <= 0:
                        break
                else:
                    data = fd.read(min(content_length, CONF['bufSize']))
                    if not data:
                        break

                    self.connection.send(data)
                    sent = len(data)

                offset += sent
                content_length -= sent

                if CONF['showSpeed']:
                    tm_b = [time.time(), offset - start]
                    delta = tm_b[0] - tm_a[0]
                    rest = end - offset + 1

                    if delta >= CONF['updateInterval'] or rest == 0:
                        speed = (tm_b[1] - tm_a[1]) / delta
                        log.info("SPD: %.2fKB/S, XFER / TODO: %d / %d bytes, ETA %d sec"
                                 % (speed / 1000, offset - start, rest, rest / speed))
                        tm_a = tm_b
        except (OSError, socket.error) as e:
            log.error("Connection dropped, %d bytes sent, reason: %s", offset - start, str(e))
Exemple #3
0
def start_server():
    server_address = ("0.0.0.0", CONF['port'])
    proxy_handler.ProxyHandler.protocol_version = "HTTP/1.1"
    run_event = threading.Event()

    httpd = ThreadingHTTPServer(server_address, proxy_handler.ProxyHandler, CONF['cache'])
    sa = httpd.socket.getsockname()
    log.info("Download directory: %s", CONF['downloadDIR'])
    log.info("Servering HTTP on %s %s %s", sa[0], "port", sa[1])
    req_count = 0

    while not run_event.isSet():
        try:
            httpd.handle_request()
            req_count += 1
            if req_count == 1000:
                log.info("Number of active threads: %s", threading.activeCount())
                req_count = 0
        except select.error, e:
            if e[0] == 4 and run_event.isSet(): pass
            else:
                log.error("Errno: %d - %s", e[0], e[1])
        except socket.error as e:
            if e.errno == 10054:
                log.error("Connection reset by peer")
            else:
                log.error(str(e))
Exemple #4
0
    def load_cache_list(self, fn):
        if not os.path.exists(fn):
            with open(fn, "w") as f:
                f.write("# format in cache.txt:\n\n")
                f.write("# replace_url->replace_file_path\n")
                f.write("# search:replace_filename->replace_file_path\n")
                f.write("# re:replace_url_regular_expression->replace_file_path\n\n")
                f.write("# if replace_file_path is relative then it will be lookup from download directory\n")
        try:
            with open(fn, "r") as f:
                for l in f:
                    l = l.decode(errors='ignore').strip()

                    if not l or l[0] == '#':
                        continue

                    if '->' in l:
                        url, fn = l.split('->')
                    else:
                        url = l
                        filename = url.split("?")[0].split("/")[-1]
                    if not fn:
                        continue

                    " fn如为相对路径名应加上下载目录路径 "
                    if not os.path.isabs(fn):
                        fn = os.path.join(CONF['downloadDIR'], fn)

                    try:
                        open(fn).close()
                        self.replace_dict[url] = fn
                    except IOError as e:
                        log.error("%s: %s", fn, str(e))

            log.info("%d local caches loaded" % (len(self.replace_dict)))
            log.debug("Dumping cache list:")
            log.debug(self.replace_dict)
        except IOError as e:
            log.error(str(e))
Exemple #5
0
    def do_GET(self, head_only=False):
        self.close_connection = 1
        self.fixPSVBrokenPath()

        log.info("GET %s", self.path)

        (scm, netloc, path, params, query, fragment) = urlparse.urlparse(self.path, 'http')

        if not path:
            path = '/'

        if scm not in ('http', 'ftp') or fragment or not netloc:
            self.send_error(400, "bad url %s" % self.path)
            return

        soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        try:
            if self.tryDownloadPath(self.path, head_only):
                return
            for url, filename in self.server.replace_dict.iteritems():
                if url.startswith('re:'):
                    r = re.compile(url[len('re:'):])

                    if r.match(self.path):
                        replace_url = self.path
                        self.getLocalCache(filename, head_only)
                        return

                if url.startswith('search:') and url[len('search:'):] in self.path:
                    self.getLocalCache(filename, head_only)
                    return

                if url == self.path:
                    self.getLocalCache(filename, head_only)
                    return

            if scm == 'http':
                if self._connect_to(netloc, soc):
                    soc.send("%s %s %s\r\n" % (self.command,
                                               urlparse.urlunparse(('', '', path,
                                                                    params, query,
                                                                    '')),
                                               self.request_version))
                    self.headers['Connection'] = 'close'
                    del self.headers['Proxy-Connection']
                    for key_val in self.headers.items():
                        soc.send("%s: %s\r\n" % key_val)
                    soc.send("\r\n")
                    self._read_write(soc)
            elif scm == 'ftp':
                # fish out user and password information
                i = netloc.find ('@')
                if i >= 0:
                    login_info, netloc = netloc[:i], netloc[i+1:]
                    try: user, passwd = login_info.split (':', 1)
                    except ValueError: user, passwd = "anonymous", None
                else: user, passwd ="anonymous", None

                ftp = ftplib.FTP (netloc)
                ftp.login (user, passwd)
                if self.command == "GET":
                    ftp.retrbinary ("RETR %s"%path, self.connection.send)
                ftp.quit ()
        except Exception as e:
            log.error("Connection dropped, reason: %s", str(e))
        finally:
            soc.close()
            self.connection.close()