def chek_pages(pages): try: for pages_url in pages: urllib.splitpasswd("*****@*****.**") urllib.splituser() code = urllib.urlopen(pages_url).getcode() print pages_url, code if code not in [200, 301]: failed_pages.append(pages_url) except socket.error, e: print "Ping Error", e
def test_splitpasswd(self): """Some of the password examples are not sensible, but it is added to confirming to RFC2617 and addressing issue4675. """ self.assertEqual(('user', 'ab'), urllib.splitpasswd('user:ab')) self.assertEqual(('user', 'a\nb'), urllib.splitpasswd('user:a\nb')) self.assertEqual(('user', 'a\tb'), urllib.splitpasswd('user:a\tb')) self.assertEqual(('user', 'a\rb'), urllib.splitpasswd('user:a\rb')) self.assertEqual(('user', 'a\fb'), urllib.splitpasswd('user:a\fb')) self.assertEqual(('user', 'a\vb'), urllib.splitpasswd('user:a\vb')) self.assertEqual(('user', 'a:b'), urllib.splitpasswd('user:a:b')) self.assertEqual(('user', 'a b'), urllib.splitpasswd('user:a b')) self.assertEqual(('user 2', 'ab'), urllib.splitpasswd('user 2:ab')) self.assertEqual(('user+1', 'a+b'), urllib.splitpasswd('user+1:a+b'))
def test_splitpasswd(self): """Some of the password examples are not sensible, but it is added to confirming to RFC2617 and addressing issue4675. """ self.assertEqual(("user", "ab"), urllib.splitpasswd("user:ab")) self.assertEqual(("user", "a\nb"), urllib.splitpasswd("user:a\nb")) self.assertEqual(("user", "a\tb"), urllib.splitpasswd("user:a\tb")) self.assertEqual(("user", "a\rb"), urllib.splitpasswd("user:a\rb")) self.assertEqual(("user", "a\fb"), urllib.splitpasswd("user:a\fb")) self.assertEqual(("user", "a\vb"), urllib.splitpasswd("user:a\vb")) self.assertEqual(("user", "a:b"), urllib.splitpasswd("user:a:b")) self.assertEqual(("user", "a b"), urllib.splitpasswd("user:a b")) self.assertEqual(("user 2", "ab"), urllib.splitpasswd("user 2:ab")) self.assertEqual(("user+1", "a+b"), urllib.splitpasswd("user+1:a+b"))
def test_splitpasswd(self): """Some of the password examples are not sensible, but it is added to confirming to RFC2617 and addressing issue4675. """ self.assertEqual(('user', 'ab'),urllib.splitpasswd('user:ab')) self.assertEqual(('user', 'a\nb'),urllib.splitpasswd('user:a\nb')) self.assertEqual(('user', 'a\tb'),urllib.splitpasswd('user:a\tb')) self.assertEqual(('user', 'a\rb'),urllib.splitpasswd('user:a\rb')) self.assertEqual(('user', 'a\fb'),urllib.splitpasswd('user:a\fb')) self.assertEqual(('user', 'a\vb'),urllib.splitpasswd('user:a\vb')) self.assertEqual(('user', 'a:b'),urllib.splitpasswd('user:a:b')) self.assertEqual(('user', 'a b'),urllib.splitpasswd('user:a b')) self.assertEqual(('user 2', 'ab'),urllib.splitpasswd('user 2:ab')) self.assertEqual(('user+1', 'a+b'),urllib.splitpasswd('user+1:a+b'))
def signon(self, url='localhost:8889', login=None, password=None, startQueue=False, verbose=False): """Sign on to RTS2 JSON server. url - JSON API URL (can include username and login)""" # try to get username (and password) from url purl = urlparse.urlsplit(url) userpass,host = urllib.splituser(purl.netloc) (userpass, netloc) = urllib.splituser(purl.netloc) if userpass is not None: (login, password) = urllib.splitpasswd(userpass) url = netloc + purl.path if purl.query: url += '?' + query if purl.fragment: url += '#' + fragment if login is None: self.verbose.set_active(verbose) self.dialog.show_all() if self.run(): self.dialog.hide() sys.exit(-1) self.dialog.hide() else: # or just create server.. createJsonServer(url, login, password, verbose=verbose) getProxy().loadJson('/api/devices') if startQueue: getProxy().startQueue() getProxy().refresh()
def __init__(self, url): """ Initialize the downloader with the specified url string """ # FIXME: support HTTPS scheme, host, path, params, query, fragment = urlparse.urlparse(url) auth, host = urllib.splituser(host) self.host, self.port = urllib.splitport(host) if not self.port: self.port = 80 self.username = self.password = None if auth: self.username, self.password = urllib.splitpasswd(auth) self.url = urlparse.urlunparse( (scheme, host, path, params, query, fragment)) self.nzbFilename = os.path.basename(path) self.tempFilename = os.path.join( Hellanzb.TEMP_DIR, tempFilename(self.TEMP_FILENAME_PREFIX) + '.nzb') # The HTTPDownloader self.downloader = None # The NZB category (e.g. 'Apps') self.nzbCategory = None # Whether or not the NZB file data is gzipped self.isGzipped = False
def signon(self, url='localhost:8889', login=None, password=None, startQueue=False, verbose=False): """Sign on to RTS2 JSON server. url - JSON API URL (can include username and login)""" # try to get username (and password) from url purl = urlparse.urlsplit(url) userpass, host = urllib.splituser(purl.netloc) (userpass, netloc) = urllib.splituser(purl.netloc) if userpass is not None: (login, password) = urllib.splitpasswd(userpass) url = netloc + purl.path if purl.query: url += '?' + query if purl.fragment: url += '#' + fragment if login is None: self.verbose.set_active(verbose) self.dialog.show_all() if self.run(): self.dialog.hide() sys.exit(-1) self.dialog.hide() else: # or just create server.. createJsonServer(url, login, password, verbose=verbose) getProxy().loadJson('/api/devices') if startQueue: getProxy().startQueue() getProxy().refresh()
def __init__(self, url): """ Initialize the downloader with the specified url string """ # FIXME: support HTTPS scheme, host, path, params, query, fragment = urlparse.urlparse(url) auth, host = urllib.splituser(host) self.host, self.port = urllib.splitport(host) if not self.port: self.port = 80 self.username = self.password = None if auth: self.username, self.password = urllib.splitpasswd(auth) self.url = urlparse.urlunparse((scheme, host, path, params, query, fragment)) self.nzbFilename = os.path.basename(path) self.tempFilename = os.path.join(Hellanzb.TEMP_DIR, tempFilename(self.TEMP_FILENAME_PREFIX) + '.nzb') # The HTTPDownloader self.downloader = None # The NZB category (e.g. 'Apps') self.nzbCategory = None # Whether or not the NZB file data is gzipped self.isGzipped = False
def ftp_open(self, req): import ftplib import mimetypes host = req.get_host() if not host: raise IOError, ('ftp error', 'no host given') host, port = splitport(host) if port is None: port = ftplib.FTP_PORT else: port = int(port) # username/password handling user, host = splituser(host) if user: user, passwd = splitpasswd(user) else: passwd = None host = unquote(host) user = unquote(user or '') passwd = unquote(passwd or '') try: host = socket.gethostbyname(host) except socket.error, msg: raise URLError(msg)
def create_auth_baton(url): """Create an authentication baton for the specified URL. :param url: URL to create auth baton for. """ import urlparse (scheme, netloc, path, _, _) = urlparse.urlsplit(url) (creds, host) = urllib.splituser(netloc) (host, port) = urllib.splitport(host) auth_config = SubversionAuthenticationConfig(scheme, host, port, path) # Specify Subversion providers first, because they use file data # rather than prompting the user. providers = [] providers += get_stock_svn_providers() providers += auth_config.get_svn_auth_providers() providers += [get_ssl_client_cert_pw_provider(1)] auth_baton = ra.Auth(providers) if creds is not None: (user, password) = urllib.splitpasswd(creds) if user is not None: auth_baton.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME, user) if password is not None: auth_baton.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_PASSWORD, password) return auth_baton
def __init__(self, uri, cookiejar=None, use_datetime=0): Transport.__init__(self, use_datetime=use_datetime) self.opener = build_opener() # Parse auth (user:passwd) from the uri urltype, rest = splittype(uri) host, rest = splithost(rest) auth, host = splituser(host) self.uri = urltype + '://' + host + rest # Handle HTTP Basic authentication if auth is not None: user, passwd = splitpasswd(auth) passwdmgr = HTTPPasswordMgrWithDefaultRealm() passwdmgr.add_password(realm=None, uri=self.uri, user=user, passwd=passwd) authhandler = HTTPBasicAuthHandler(passwdmgr) self.opener.add_handler(authhandler) # Handle HTTP Cookies if cookiejar is not None: self.opener.add_handler(HTTPCookieProcessor(cookiejar))
def parse_address_info(cls, server_addr="nats://*****:*****@127.0.0.1:4222"): '''\ parse the metadata nats server uri; Params: ===== addr: nats server address; Returns: ===== user: username to login nats server; pswd: password to login nats server; host: ip address of nats server; port: port of nats server ''' if type(server_addr) is not str: raise NotImplementException protocol, after_split = urllib.splittype(server_addr) if not protocol == "nats": raise NotImplementException auth_len = len(server_addr.split('@')) if auth_len > 1: auth, after_split = urllib.splituser(after_split) user_raw, pswd = urllib.splitpasswd(auth) user = user_raw.lstrip("/") _, after_split = urllib.splithost(after_split) host, port = urllib.splitport(after_split) else: user = pswd = None host, port = urllib.splitport(after_split) return user, pswd, host, int(port)
def ftp_open(self, req): import ftplib import mimetypes host = req.get_host() if not host: raise URLError('ftp error: no host given') host, port = splitport(host) if port is None: port = ftplib.FTP_PORT else: port = int(port) # username/password handling user, host = splituser(host) if user: user, passwd = splitpasswd(user) else: passwd = None host = unquote(host) user = unquote(user or '') passwd = unquote(passwd or '') try: host = socket.gethostbyname(host) except socket.error, msg: raise URLError(msg)
def __init__(self, baseUri, headers=None, maxClients=None, maxConnections=None): self._headers = headers or HTTPHeaders() self._user = None self._passwd = None baseUri = baseUri.rstrip('/') self._scheme, loc, self._path, query, frag = urlparse.urlsplit(baseUri) userpass, self._hostport = urllib.splituser(loc) if userpass: self._user, self._passwd = urllib.splitpasswd(userpass) self._baseUri = urlparse.urlunsplit((self._scheme, self._hostport, self._path, None, None)) if self._scheme not in ('http', 'https'): raise ValueError(self._scheme) self._dispatcher = RequestDispatcher(maxClients=maxClients, maxConnections=maxConnections) self._queryFragment = urlparse.urlunsplit(('', '', '', query, frag))
def __init__(self, url): self.scheme, self.netloc, self.path, query, self.fragment = \ urlparse.urlsplit(url) self.user, self.host = urllib.splituser(self.netloc) if not self.user: try: # get userid from current user self.user = pwd.getpwuid(os.getuid())[0] except KeyError: self.user = "******" self.user = self.user or "" self.userid = self.username = self.user # aliases self.adapter = self.scheme self.host, self.port = urllib.splitport(self.host) self.user, self.password = urllib.splitpasswd(self.user or "") self.password = self.password or "" self.path = self.path[1:] self.database = self.path if query: try: self.query = dict([ (k, urllib.unquote_plus(v)) for k, v in [e.split("=", 1) for e in self.query.split("&", 1)] ]) except ValueError, e: raise ValueError, "Query part in url cannot be parsed: %r (%s)" % ( self.query, e)
def get_user_password (self): """Get tuple (user, password) from configured authentication. Both user and password can be None. """ if self.userinfo: # URL itself has authentication info return urllib.splitpasswd(self.userinfo) return self.aggregate.config.get_user_password(self.url)
def get_user_password(self): """Get tuple (user, password) from configured authentication. Both user and password can be None. """ if self.userinfo: # URL itself has authentication info return urllib.splitpasswd(self.userinfo) return self.aggregate.config.get_user_password(self.url)
def __init__(self, uri, username=None, password=None, *args, **kwargs): xmlrpclib.ServerProxy.__init__(self, uri, *args, **kwargs) # Hide password userpass, hostport = urllib.splituser(self.__host) if userpass and not username: self.__host = hostport username, password = urllib.splitpasswd(userpass) if username: password = util.ProtectedString(urllib.quote(password)) self.__host = ProtectedTemplate('${user}:${password}@${host}', user=username, password=password, host=self.__host)
def ftp_open(self, req): import ftplib import mimetypes host = req.get_host() if not host: raise URLError('ftp error: no host given') host, port = splitport(host) if port is None: port = ftplib.FTP_PORT else: port = int(port) user, host = splituser(host) if user: user, passwd = splitpasswd(user) else: passwd = None host = unquote(host) user = user or '' passwd = passwd or '' try: host = socket.gethostbyname(host) except socket.error as msg: raise URLError(msg) path, attrs = splitattr(req.get_selector()) dirs = path.split('/') dirs = map(unquote, dirs) dirs, file = dirs[:-1], dirs[-1] if dirs and not dirs[0]: dirs = dirs[1:] try: fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' for attr in attrs: attr, value = splitvalue(attr) if attr.lower() == 'type' and value in ('a', 'A', 'i', 'I', 'd', 'D'): type = value.upper() fp, retrlen = fw.retrfile(file, type) headers = '' mtype = mimetypes.guess_type(req.get_full_url())[0] if mtype: headers += 'Content-type: %s\n' % mtype if retrlen is not None and retrlen >= 0: headers += 'Content-length: %d\n' % retrlen sf = StringIO(headers) headers = mimetools.Message(sf) return addinfourl(fp, headers, req.get_full_url()) except ftplib.all_errors as msg: raise URLError, 'ftp error: %s' % msg, sys.exc_info()[2] return
def parse_backend(url): def unescape(s): return urllib.unquote(s.replace('+', ' ')) try: proto, u = urllib.splittype(url) auth, path = urllib.splithost(u) user, host = urllib.splituser(auth) user, password = urllib.splitpasswd(user) host, _ = urllib.splitport(host) _, query = urllib.splitquery(path) args = query.split('&') hot_user = None hot_pass = None level = 0 mode = -1 for arg in args: k, v = arg.split('=', 2) v = unescape(v) if k == 'hot_user': hot_user = v elif k == 'hot_pass': hot_pass = v elif k == 'level': level = parse_level(v) elif k == 'mode': mode = parse_mode(v) elif k == 'extra': hot_addr, extra = parse_extra(v) if hot_user != None and hot_addr == None: hot_addr = host Backend = namedtuple('Backend', [ 'address', 'user', 'password', 'hot_addr', 'hot_user', 'hot_pass', 'extra', 'level', 'mode' ]) return Backend(address=host, user=user, password=password, hot_addr=hot_addr, hot_user=hot_user, hot_pass=hot_pass, extra=extra, level=level, mode=mode) except Exception: logging.getLogger('mwtm_executor').error('bad backend: %s', url, exc_info=True) _exit(1)
def ParseUrl(value): type = "" host = "" port = "" path = "" username = "" password = "" type, value = splittype(value or "") value, path = splithost(value or "") usps, value = splituser(value or "") if usps: username, password = splitpasswd(usps) host, port = splitnport(value or "", None) return type or "", host or "", str( port) if port else "", path or "", username or "", password or ""
def check_args(options, args): if not args: return # someway to test if it is a SIP URI if 'sip:' in args[0]: setattr(options, 'target_uri', args[0]) userpass, hostport = urllib.splituser(args[0]) if hostport[:4] == 'sip:': hostport = hostport[4:] host, port = urllib.splitnport(hostport, 5060) # XXX Overwrites! setattr(options, 'target_addr', (host, port)) if userpass: user, passwd = urllib.splitpasswd(userpass) user and setattr(options, 'user', user) passwd and setattr(options, 'passwd', passwd)
def _getconnection(netloc): """Return a FTP connection object to the specified server.""" # NOTE: this method is not thread safe if _ftpconnections.has_key(netloc): return _ftpconnections[netloc] # split url into useful parts (userpass, host) = urllib.splituser(netloc) if userpass is not None: (user, passwd) = urllib.splitpasswd(userpass) else: (user, passwd) = ('anonymous', '') (host, port) = urllib.splitnport(host, ftplib.FTP_PORT) # initialize a new connection ftp = ftplib.FTP() debugio.debug('schemes.ftp._getconnection(): CONNECT: '+ftp.connect(host, port)) debugio.debug('schemes.ftp._getconnection(): LOGIN: '+ftp.login(user, passwd)) _ftpconnections[netloc] = ftp return ftp
def _parse_proxy(proxy): scheme, r_scheme = splittype(proxy) if not r_scheme.startswith('/'): scheme = None authority = proxy else: if not r_scheme.startswith('//'): raise ValueError('proxy URL with no authority: %r' % proxy) end = r_scheme.find('/', 2) if end == -1: end = None authority = r_scheme[2:end] userinfo, hostport = splituser(authority) if userinfo is not None: user, password = splitpasswd(userinfo) else: user = password = None return (scheme, user, password, hostport)
def urlSplit(url, defaultPort = None): """A function to split a URL in the format <scheme>://<user>:<pass>@<host>:<port>/<path>;<params>#<fragment> into a tuple (<scheme>, <user>, <pass>, <host>, <port>, <path>, <params>, <fragment>) Any missing pieces (user/pass) will be set to None. If the port is missing, it will be set to defaultPort; otherwise, the port should be a numeric value. """ scheme, netloc, path, query, fragment = urlparse.urlsplit(url) userpass, hostport = urllib.splituser(netloc) host, port = urllib.splitnport(hostport, None) if userpass: user, passwd = urllib.splitpasswd(userpass) else: user, passwd = None, None return scheme, user, passwd, host, port, path, \ query or None, fragment or None
def urlSplit(url, defaultPort=None): """A function to split a URL in the format <scheme>://<user>:<pass>@<host>:<port>/<path>;<params>#<fragment> into a tuple (<scheme>, <user>, <pass>, <host>, <port>, <path>, <params>, <fragment>) Any missing pieces (user/pass) will be set to None. If the port is missing, it will be set to defaultPort; otherwise, the port should be a numeric value. """ scheme, netloc, path, query, fragment = urlparse.urlsplit(url) userpass, hostport = urllib.splituser(netloc) host, port = urllib.splitnport(hostport, None) if userpass: user, passwd = urllib.splitpasswd(userpass) else: user, passwd = None, None return scheme, user, passwd, host, port, path, \ query or None, fragment or None
def __init__(self, url): type, uri = urllib.splittype(url) #if type not in ("http", "https"): # raise IOError, "unsupported XML-RPC protocol" self.host, self.url = urllib.splithost(uri) if self.url == "": self.url = "/" self.user = self.password = None self.user, self.host = urllib.splituser(self.host) try: self.user, self.password = urllib.splitpasswd(self.user) except TypeError: pass self.host, self.port = urllib.splitport(self.host) self.port = int(self.port) self.secure = type == 'https'
def build_http_request(self, host, handler, request_body): token = None user = None auth, host = urllib.splituser(host) if auth: user, token = urllib.splitpasswd(auth) url = self._scheme + "://" + host + handler if user is not None and token is None: token = self.auth_backend.get_token_for_endpoint(user, url) if token is None: raise LavaCommandError( "Username provided but no token found.") request = urllib2.Request(url, request_body) request.add_header("Content-Type", "text/xml") if token: auth = base64.b64encode(urllib.unquote(user + ':' + token)) request.add_header("Authorization", "Basic " + auth) return request
def parse_remote_repository (uri, ) : _pa = urlparse.urlsplit(uri) (_a_user, _a_host, ) = urllib.splituser(_pa.netloc, ) _user, _password = None, None if _a_user : (_user, _password, ) = urllib.splitpasswd(_a_user, ) _defaut_port = _pa.scheme.lower() in ("svn+ssh", "ssh", ) and 22 or None (_host, _port, ) = urllib.splitnport(_a_host, defport=_defaut_port, ) return dict( host=_host, port=_port, scheme=_pa.scheme.lower(), user=_user, password=_password, path=os.path.normpath( _pa.path.strip().startswith("/") and _pa.path or ("/" + _pa.path) ), )
def parse_backend(url): def unescape(s): return urllib.unquote(s.replace('+', ' ')) try: proto, u = urllib.splittype(url) auth, path = urllib.splithost(u) user, host = urllib.splituser(auth) user, password = urllib.splitpasswd(user) host, _ = urllib.splitport(host) _, query = urllib.splitquery(path) args = query.split('&') hot_user = None hot_pass = None level = 0 mode = -1 for arg in args: k, v = arg.split('=', 2) v = unescape(v) if k == 'hot_user': hot_user = v elif k == 'hot_pass': hot_pass = v elif k == 'level': level = parse_level(v) elif k == 'mode': mode = parse_mode(v) elif k == 'extra': hot_addr, extra = parse_extra(v) if hot_user != None and hot_addr == None: hot_addr = host Backend = namedtuple('Backend', ['address', 'user', 'password', 'hot_addr', 'hot_user', 'hot_pass', 'extra', 'level', 'mode']) return Backend(address=host, user=user, password=password, hot_addr=hot_addr, hot_user=hot_user, hot_pass=hot_pass, extra=extra, level=level, mode=mode) except Exception: logging.getLogger('mwtm_executor').error('bad backend: %s', url, exc_info=True) _exit(1)
def parse_uri(uri, **kwargs): scheme, rest = urllib.splittype(uri) host, rest = urllib.splithost(rest) user, host = urllib.splituser(host) if user: username, password = urllib.splitpasswd(user) else: username = password = None host, port = urllib.splitnport(host) path, query = urllib.splitquery(rest) if query: kwargs.update(dict(cgi.parse_qsl(query))) return dict( scheme=scheme, host=host, username=username, password=password, port=port, path=path, query=kwargs)
def _getBaseServerUrlData(self): """ Fetch serverUrl from ~/.rbuilderrc if it exists and is specified; removes user and password from the URL and returns them separately. @return serverUrl, user, password """ serverUrl = self._getBaseServerUrl() if not serverUrl: return (None, None, None) scheme, rest = serverUrl.split(':', 1) host = urllib.splithost(rest)[0] user = urllib.splituser(host)[0] if user: user, password = urllib.splitpasswd(user) else: password = None if password: serverUrl = serverUrl.replace(':%s' % password, '', 1) if user: serverUrl = serverUrl.replace('%s@' % user, '', 1) return serverUrl, user, password
def _getBaseServerUrlData(self): """ Fetch serverUrl from ~/.rbuilderrc if it exists and is specified; removes user and password from the URL and returns them separately. @return serverUrl, user, password """ serverUrl = self._getBaseServerUrl() if not serverUrl: return (None, None, None) scheme, rest = serverUrl.split(":", 1) host = urllib.splithost(rest)[0] user = urllib.splituser(host)[0] if user: user, password = urllib.splitpasswd(user) else: password = None if password: serverUrl = serverUrl.replace(":%s" % password, "", 1) if user: serverUrl = serverUrl.replace("%s@" % user, "", 1) return serverUrl, user, password
def _parseURI(uri): protocol, request = urllib.splittype(uri) user, password, port = None, None, None host, path = urllib.splithost(request) if host: # Python < 2.7 have a problem - splituser() calls unquote() too early #user, host = urllib.splituser(host) if '@' in host: user, host = host.split('@', 1) if user: user, password = [ x and urllib.unquote(x) or None for x in urllib.splitpasswd(user) ] host, port = urllib.splitport(host) if port: port = int(port) elif host == '': host = None # hash-tag is splitted but ignored path, tag = urllib.splittag(path) path, query = urllib.splitquery(path) path = urllib.unquote(path) if (os.name == 'nt') and (len(path) > 2): # Preserve backward compatibility with URIs like /C|/path; # replace '|' by ':' if path[2] == '|': path = "%s:%s" % (path[0:2], path[3:]) # Remove leading slash if (path[0] == '/') and (path[2] == ':'): path = path[1:] args = {} if query: for name, value in parse_qsl(query): args[name] = value return user, password, host, port, path, args
def ftp_open(self, req): host = req.get_host() if not host: raise IOError, ("ftp error", "no host given") host, port = splitport(host) if port is None: port = ftplib.FTP_PORT # username/password handling user, host = splituser(host) if user: user, passwd = splitpasswd(user) else: passwd = None host = unquote(host) user = unquote(user or "") passwd = unquote(passwd or "") try: host = socket.gethostbyname(host) except socket.error, msg: raise URLError(msg)
def parse_url(url, default_port=None): ''' Parse url in the following form: PROTO://[USER:[:PASSWD]@]HOST[:PORT][/PATH[;ATTR][?QUERY]] A tuple containing (proto, user, passwd, host, port, path, tag, attrs, query) is returned, where `attrs' is a tuple containing ('attr1=value1', 'attr2=value2', ...) ''' proto, user, passwd, host, port, path, tag, attrs, query = (None, ) * 9 try: proto, tmp_host = urllib.splittype(url) tmp_host, tmp_path = urllib.splithost(tmp_host) tmp_user, tmp_host = urllib.splituser(tmp_host) if tmp_user: user, passwd = urllib.splitpasswd(tmp_user) host, port = urllib.splitport(tmp_host) port = int(port) if port else default_port tmp_path, query = urllib.splitquery(tmp_path) tmp_path, attrs = urllib.splitattr(tmp_path) path, tag = urllib.splittag(tmp_path) except Exception, err: raise Exception('parse_db_url error - {0}'.format(str(err)))
def __init__(self, url, headers = None): self.scheme = None self.user = None self.passwd = None self.hostport = None self.path = None self.headers = headers or {} urltype, url = urllib.splittype(url) if urltype: self.scheme = urltype.lower() host, self.path = urllib.splithost(url) if host: user_passwd, self.hostport = urllib.splituser(host) if user_passwd: self.user, self.passwd = urllib.splitpasswd(user_passwd) if self.scheme not in ['http', 'https']: raise ValueError(self.scheme) self._connection = None
def __init__(self, url): self.scheme, self.netloc, self.path, query, self.fragment = \ urlparse.urlsplit(url) self.user, self.host = urllib.splituser(self.netloc) if not self.user: try: # get userid from current user self.user = pwd.getpwuid(os.getuid())[0] except KeyError: self.user = "******" self.user = self.user or "" self.userid = self.username = self.user # aliases self.adapter = self.scheme self.host, self.port = urllib.splitport(self.host) self.user, self.password = urllib.splitpasswd(self.user or "") self.password = self.password or "" self.path = self.path[1:] self.database = self.path if query: try: self.query = dict([ (k,urllib.unquote_plus(v)) for k,v in [e.split("=", 1) for e in self.query.split("&", 1)] ]) except ValueError, e: raise ValueError, "Query part in url cannot be parsed: %r (%s)" % (self.query, e)
def parse_proxy(proxy): """ _parse_proxy from urllib """ scheme, r_scheme = splittype(proxy) if not r_scheme.startswith("/"): # authority scheme = None authority = proxy else: # URL if not r_scheme.startswith("//"): raise ValueError("proxy URL with no authority: %r" % proxy) # We have an authority, so for RFC 3986-compliant URLs (by ss 3. # and 3.3.), path is empty or starts with '/' end = r_scheme.find("/", 2) if end == -1: end = None authority = r_scheme[2:end] userinfo, hostport = splituser(authority) if userinfo is not None: user, password = splitpasswd(userinfo) else: user = password = None return scheme, user, password, hostport
def _parseURI(uri): protocol, request = urllib.splittype(uri) user, password, port = None, None, None host, path = urllib.splithost(request) if host: # Python < 2.7 have a problem - splituser() calls unquote() too early #user, host = urllib.splituser(host) if '@' in host: user, host = host.split('@', 1) if user: user, password = [x and urllib.unquote(x) or None for x in urllib.splitpasswd(user)] host, port = urllib.splitport(host) if port: port = int(port) elif host == '': host = None # hash-tag is splitted but ignored path, tag = urllib.splittag(path) path, query = urllib.splitquery(path) path = urllib.unquote(path) if (os.name == 'nt') and (len(path) > 2): # Preserve backward compatibility with URIs like /C|/path; # replace '|' by ':' if path[2] == '|': path = "%s:%s" % (path[0:2], path[3:]) # Remove leading slash if (path[0] == '/') and (path[2] == ':'): path = path[1:] args = {} if query: for name, value in parse_qsl(query): args[name] = value return user, password, host, port, path, args
def sane_config(options): if not options.target_uri and not options.to_uri: options.target_uri = options.to_uri = '"siping" <sip:siping@localhost>' elif options.target_uri and not options.to_uri: options.to_uri = options.target_uri elif options.to_uri and not options.target_uri: options.target_uri = options.to_uri #if not options.target_uri: options.target_uri = options.to_uri if not options.from_uri: options.from_uri = options.to_uri if not options.contacts_uri: options.contacts_uri.append( options.from_uri ) if not options.cseq_method: options.cseq_method = options.method if not hasattr(options, 'target_addr'): hp = options.to_uri.find('sip:') hp = options.to_uri[hp+4:] if '@' in hp: up, hp = urllib.splituser(hp) if hp[-1] == '>': hp = hp[:-1] h, p = urllib.splitnport(hp, 5060) if type(p) != int: p = int(p) setattr(options, 'target_addr', (h, p)) if up: u, p = urllib.splitpasswd(up) setattr(options, 'username', u) setattr(options, 'password', p) if not options.target_addr[1]: # no port options.target_addr = (options.target_addr[0], 5060)
def parse (url): import urllib import socket db_conf = {}; start_idx = url.find ('mysql://'); if start_idx != 0: raise Exception ("bad mysql format (%s)" %url); try: start_idx = len ('mysql://'); end_idx = url.find ('/', start_idx); if end_idx >= 0: tidx = url.find ('/', end_idx+1); if tidx < 0: db_conf['db'] = url[end_idx+1:]; db_conf['tbl'] = None; else: db_conf['db'] = url[end_idx+1:tidx]; db_conf['tbl'] = url[tidx+1:] else: db_conf['db'] = None; db_conf['tbl'] = None; passwd, port = urllib.splituser (url[start_idx:end_idx]); db_conf['user'], db_conf['pass'] = urllib.splitpasswd (passwd); host, db_port = urllib.splitport (port); #try: # socket.inet_aton (host) #except socket.error: # raise db_conf['host'] = host if db_port is not None: db_conf['port'] = int (db_port); else: db_conf['port'] = 3306 except Exception, msg: raise Exception ("bad mysql format (%s), %s" % (url, msg));
def sane_config(options): if not options.target_uri and not options.to_uri: options.target_uri = options.to_uri = '"siping" <sip:siping@localhost>' elif options.target_uri and not options.to_uri: options.to_uri = options.target_uri elif options.to_uri and not options.target_uri: options.target_uri = options.to_uri #if not options.target_uri: options.target_uri = options.to_uri if not options.from_uri: options.from_uri = options.to_uri if not options.contacts_uri: options.contacts_uri.append(options.from_uri) if not options.cseq_method: options.cseq_method = options.method if not hasattr(options, 'target_addr'): hp = options.to_uri.find('sip:') hp = options.to_uri[hp + 4:] if '@' in hp: up, hp = urllib.splituser(hp) if hp[-1] == '>': hp = hp[:-1] h, p = urllib.splitnport(hp, 5060) if type(p) != int: p = int(p) setattr(options, 'target_addr', (h, p)) if up: u, p = urllib.splitpasswd(up) setattr(options, 'username', u) setattr(options, 'password', p) if not options.target_addr[1]: # no port options.target_addr = (options.target_addr[0], 5060)
def parse(url): import urllib import socket db_conf = {} start_idx = url.find('mysql://') if start_idx != 0: raise Exception("bad mysql format (%s)" % url) try: start_idx = len('mysql://') end_idx = url.find('/', start_idx) if end_idx >= 0: tidx = url.find('/', end_idx + 1) if tidx < 0: db_conf['db'] = url[end_idx + 1:] db_conf['tbl'] = None else: db_conf['db'] = url[end_idx + 1:tidx] db_conf['tbl'] = url[tidx + 1:] else: db_conf['db'] = None db_conf['tbl'] = None passwd, port = urllib.splituser(url[start_idx:end_idx]) db_conf['user'], db_conf['pass'] = urllib.splitpasswd(passwd) host, db_port = urllib.splitport(port) #try: # socket.inet_aton (host) #except socket.error: # raise db_conf['host'] = host if db_port is not None: db_conf['port'] = int(db_port) else: db_conf['port'] = 3306 except Exception, msg: raise Exception("bad mysql format (%s), %s" % (url, msg))
def _parse_proxy(proxy): """Return (scheme, user, password, host/port) given a URL or an authority. If a URL is supplied, it must have an authority (host:port) component. According to RFC 3986, having an authority component means the URL must have two slashes after the scheme: >>> _parse_proxy('file:/ftp.example.com/') Traceback (most recent call last): ValueError: proxy URL with no authority: 'file:/ftp.example.com/' The first three items of the returned tuple may be None. Examples of authority parsing: >>> _parse_proxy('proxy.example.com') (None, None, None, 'proxy.example.com') >>> _parse_proxy('proxy.example.com:3128') (None, None, None, 'proxy.example.com:3128') The authority component may optionally include userinfo (assumed to be username:password): >>> _parse_proxy('joe:[email protected]') (None, 'joe', 'password', 'proxy.example.com') >>> _parse_proxy('joe:[email protected]:3128') (None, 'joe', 'password', 'proxy.example.com:3128') Same examples, but with URLs instead: >>> _parse_proxy('http://proxy.example.com/') ('http', None, None, 'proxy.example.com') >>> _parse_proxy('http://proxy.example.com:3128/') ('http', None, None, 'proxy.example.com:3128') >>> _parse_proxy('http://*****:*****@proxy.example.com/') ('http', 'joe', 'password', 'proxy.example.com') >>> _parse_proxy('http://*****:*****@proxy.example.com:3128') ('http', 'joe', 'password', 'proxy.example.com:3128') Everything after the authority is ignored: >>> _parse_proxy('ftp://*****:*****@proxy.example.com/rubbish:3128') ('ftp', 'joe', 'password', 'proxy.example.com') Test for no trailing '/' case: >>> _parse_proxy('http://*****:*****@proxy.example.com') ('http', 'joe', 'password', 'proxy.example.com') """ scheme, r_scheme = splittype(proxy) if not r_scheme.startswith("/"): # authority scheme = None authority = proxy else: # URL if not r_scheme.startswith("//"): raise ValueError("proxy URL with no authority: %r" % proxy) # We have an authority, so for RFC 3986-compliant URLs (by ss 3. # and 3.3.), path is empty or starts with '/' end = r_scheme.find("/", 2) if end == -1: end = None authority = r_scheme[2:end] userinfo, hostport = splituser(authority) if userinfo is not None: user, password = splitpasswd(userinfo) else: user = password = None return scheme, user, password, hostport
def get_repository(self, repos_type, repos_dir, authname): assert repos_type == 'perforce' import urllib urltype, url = urllib.splittype(repos_dir) assert urltype == 'p4' or url == 'p4' options = dict(self.config.options('perforce')) if urltype != None: machine, path_query = urllib.splithost(url) user_passwd, host_port = urllib.splituser(machine) user, password = urllib.splitpasswd(user_passwd) self._update_option(options, 'port', host_port) self._update_option(options, 'password', password) self._update_option(options, 'user', user) path, query = urllib.splitquery(path_query) if path and path != '': for attr in self._splitattributes(query): key, val = urllib.splitvalue(attr) self._update_option(options, key, val) self._update_option(options, 'path', path) self.log.debug("get_repository options : %s" % (options)) if 'port' not in options: raise TracError( message="Missing 'port' value in [perforce] config section.", title="TracPerforce configuration error", ) # Try to connect to the Perforce server from perforce import Connection, ConnectionFailed p4 = Connection( port=options['port'], api='58', # Limit to 2005.2 behaviour ) try: from trac import __version__ as tracVersion p4.connect(prog='Trac', version=tracVersion) except ConnectionFailed: raise TracError( message="Could not connect to Perforce repository.", title="Perforce connection error", ) if 'user' not in options: raise TracError( message="Missing 'user' value in [perforce] config section.", title="Perforce configuration error", ) p4.user = options['user'] if 'password' in options: p4.password = options['password'] else: p4.password = '' if 'unicode' in options: if options['unicode'] == '1': p4.charset = 'utf8' elif options['unicode'] == '0': p4.charset = 'none' else: raise TracError( message="Invalid 'unicode' value in [perforce] config " \ "section.", title="Perforce configuration error", ) else: p4.charset = 'none' if 'language' in options: p4.language = options['language'] else: p4.language = '' jobPrefixLength = 3 # default value because by default prefix is 'job' if 'job_prefix' in options: jobPrefixLength = len(options['job_prefix']) p4.client = '' repos = PerforceRepository(p4, None, self.log, jobPrefixLength) crepos = CachedRepository(self.env.get_db_cnx(), repos, None, self.log) return crepos
def __init__(self, url, method, params): Assert(method == 'GET') netloc, path = splithost(url) if not netloc: raise IOError, ('ftp error', 'no host given') host, port = splitport(netloc) user, host = splituser(host) if user: user, passwd = splitpasswd(user) else: passwd = None host = socket.gethostbyname(host) if port: try: port = string.atoi(port) except string.atoi_error: raise IOError, ('ftp error', 'bad port') else: port = ftplib.FTP_PORT path, attrs = splitattr(path) self.url = "ftp://%s%s" % (netloc, path) dirs = string.splitfields(path, '/') dirs, file = dirs[:-1], dirs[-1] self.content_length = None if not file: self.content_type, self.content_encoding = None, None type = 'd' else: self.content_type, self.content_encoding = app.guess_type(file) if self.content_encoding: type = 'i' elif self.content_type and self.content_type[:5] == 'text/': type = 'a' elif file[-1] == '/': type = 'd' else: type = 'i' if dirs and not dirs[0]: dirs = dirs[1:] key = (user, host, port, string.joinfields(dirs, '/')) self.debuglevel = None try: if not ftpcache.has_key(key): ftpcache[key] = [] for attr in attrs: [attr, value] = map(string.lower, splitvalue(attr)) if attr == 'type' and value in ('a', 'i', 'd'): type = value elif attr == 'debug': try: self.debuglevel = string.atoi(value) except string.atoi_error: pass candidates = ftpcache[key] for cand in candidates: if not cand.busy(): break else: cand = ftpwrapper(user, passwd, host, port, dirs, self.debuglevel) candidates.append(cand) # XXX Ought to clean the cache every once in a while self.cand = cand self.sock, self.isdir = cand.retrfile(file, type) self.content_length = cand.content_length except ftplib.all_errors, msg: raise IOError, ('ftp error', msg)
def downloadFileUsingURLLib(url, workingdir=os.getcwd(), dirPrefix=None, verbose=False, timeout=60, destFile=None): global logger global passwordManager global currentDownloadRetryCount import errno if currentDownloadRetryCount <= 0: sys.stderr.write( 'Error: reaching maximum retry for downloading.\nExiting.\n') logger.log(logging.DEBUG, 'program exists with error code 1\n') sys.exit(1) else: currentDownloadRetryCount = currentDownloadRetryCount - 1 user = getUserFromPasswordManager(logger) username = options.username if (user != None): username = user.getUsername() cookiesFilePath = getCookiesFilePath(url, username) logger.log(logging.DEBUG, 'Starting downloading file using urllib2') logger.log(logging.DEBUG, 'url=' + url) logger.log(logging.DEBUG, 'workingdir=' + workingdir) if dirPrefix != None: logger.log(logging.DEBUG, 'dirPrefix=' + dirPrefix) try: cj = cookielib.MozillaCookieJar() if os.path.exists(cookiesFilePath): cj.load(cookiesFilePath) except cookielib.LoadError: cj = cookielib.LWPCookieJar() try: if os.path.exists(cookiesFilePath): cj.load(cookiesFilePath) except cookielib.LoadError: pass opener = None scheme, sel_url = urllib.splittype(url) if scheme.lower() == 'ftp': import re valid_ftp_url = re.compile( '^ftp://([A-Za-z0-9_-]+:[A-Za-z0-9_-]+@)?(.*)$') match_valid_ftp = valid_ftp_url.match(url) invalid_ftp_msg = "Invalid ftp url: %s. Expected: ftp://[user:password@]host/path_to_file" % url if match_valid_ftp is None: raise Exception(invalid_ftp_msg) url_without_scheme = sel_url[2:] if sel_url is None: raise Exception(invalid_ftp_msg) sel_host, sel_path = splithost(sel_url) if sel_host is None: raise Exception(invalid_ftp_msg) sel_user, host = splituser(sel_host) if not sel_user is None: sel_user, sel_password = splitpasswd(sel_user) if not sel_user is None and len( sel_user) > 0 and not sel_password is None and len( sel_password) > 0: # reconstruct url url = "ftp://%s:%s@%s%s" % (sel_user, sel_password, host, sel_path) if user == None: user = User(sel_user, sel_password, url) logger.log( logging.DEBUG, 'Adding user: '******' into content password manager') else: user.setUsername(sel_user) user.setPassword(sel_password) logger.log( logging.DEBUG, 'Updating user credential for: ' + user.getUsername() + ' in content password manager') else: if not user is None: url = "ftp://%s:%s@%s%s" % (user.getUsername(), user.getPassword(), host, sel_path) if user != None: user.setRetryCount(user.getRetryCount() + 1) passman = urllib2.HTTPPasswordMgrWithDefaultRealm() passman.add_password(None, url, user.getUsername(), user.getPassword()) authhandler = urllib2.HTTPBasicAuthHandler(passman) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj), authhandler) else: opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) r = None localFile = None try: if verbose: print '\tfetching ' + url + '...' # timeout after 60 seconds of blocking operations like the connection attempt r = opener.open(url, timeout=timeout) if hasattr(options, 'checkuri') and options.checkuri: if r.code == 200: sys.exit(0) if not destFile: filename = url.split('/')[-1] else: filename = destFile if filename.strip() == '': sys.stderr.write( 'Error: URI does not contain the filename for downloading.\nExiting.\n' ) logger.log(logging.DEBUG, 'program exists with error code 1\n') sys.exit(1) if dirPrefix != None: filenamePath = os.path.join(workingdir, dirPrefix, filename) else: filenamePath = os.path.join(workingdir, filename) filenamePath = os.path.abspath(filenamePath) logger.log(logging.DEBUG, 'downloading file to ' + filenamePath) dirname = os.path.dirname(filenamePath) if not os.path.exists(dirname): os.makedirs(dirname) localFile = open(filenamePath, 'w') localFile.write(r.read()) cj.save(cookiesFilePath) return filenamePath except urllib2.HTTPError, e: if hasattr(options, 'checkuri') and options.checkuri: sys.exit(1) if hasattr(e, 'code') and e.code == 401: if user != None and user.getUri() == url: sys.stderr.write( 'Error: Invalid username and password combination for: ' + url + '\n') if user.getRetryCount() > 3: sys.stderr.write( 'Error: Max authentication retry count reached.\nExiting.\n' ) logger.log(logging.DEBUG, 'program exists with error code 2\n') sys.exit(2) if (hasattr(options, 'noninteractive') and options.noninteractive): sys.stderr.write('Error: Authentication failed. Exiting.\n') logger.log(logging.DEBUG, 'program exists with error code 2\n') sys.exit(2) else: username = raw_input('Please enter username: '******'Please enter password: '******'Adding user: '******' into content password manager') passwordManager.addUser(user) else: user.setUsername(username) user.setPassword(password) logger.log( logging.DEBUG, 'Updating user: '******' in content password manager') passwordManager.saveToken() return downloadFileUsingURLLib(url, workingdir, dirPrefix, timeout, destFile) else: if e.code != 404 or (hasattr(options, 'missingok') and not options.missingok and e.code == 404): sys.stderr.write('Receive HTTP Error for: ' + url + '\n') if hasattr(e, 'code'): sys.stderr.write('error code: ' + str(e.code) + '\n') if hasattr(e, 'msg'): sys.stderr.write('error message: ' + str(e.msg) + '\n') if hasattr(e, 'reason'): sys.stderr.write('error reason: ' + (e.reason) + '\n') sys.stderr.write('Exiting.\n') logger.log(logging.DEBUG, 'program exists with error code 3\n') sys.exit(3) elif e.code == 404 and (hasattr(options, 'missingok') and options.missingok) and verbose: logger.log(logging.WARNING, 'Failed to download file. File not found ' + url)
def smb_open(self, req): global USE_NTLM, MACHINE_NAME host = req.get_host() if not host: raise urllib2.URLError('SMB error: no host given') host, port = splitport(host) if port is None: port = 139 else: port = int(port) # username/password handling user, host = splituser(host) if user: user, passwd = splitpasswd(user) else: passwd = None host = unquote(host) user = user or '' passwd = passwd or '' myname = MACHINE_NAME or self.generateClientMachineName() n = NetBIOS() names = n.queryIPForName(host) if names: server_name = names[0] else: raise urllib2.URLError( 'SMB error: Hostname does not reply back with its machine name' ) path, attrs = splitattr(req.get_selector()) if path.startswith('/'): path = path[1:] dirs = path.split('/') dirs = map(unquote, dirs) service, path = dirs[0], '/'.join(dirs[1:]) try: conn = SMBConnection(user, passwd, myname, server_name, use_ntlm_v2=USE_NTLM) conn.connect(host, port) if req.has_data(): data_fp = req.get_data() filelen = conn.storeFile(service, path, data_fp) headers = "Content-length: 0\n" fp = StringIO("") else: fp = self.createTempFile() file_attrs, retrlen = conn.retrieveFile(service, path, fp) fp.seek(0) headers = "" mtype = mimetypes.guess_type(req.get_full_url())[0] if mtype: headers += "Content-type: %s\n" % mtype if retrlen is not None and retrlen >= 0: headers += "Content-length: %d\n" % retrlen sf = StringIO(headers) headers = mimetools.Message(sf) return addinfourl(fp, headers, req.get_full_url()) except Exception, ex: raise urllib2.URLError, ('smb error: %s' % ex), sys.exc_info()[2]