def chek_pages(pages):
try:
for pages_url in pages:
urllib.splitpasswd("*****@*****.**")
urllib.splituser()
code = urllib.urlopen(pages_url).getcode()
print pages_url, code
if code not in [200, 301]:
failed_pages.append(pages_url)
except socket.error, e:
print "Ping Error", e
def test_splitpasswd(self):
"""Some of the password examples are not sensible, but it is added to
confirming to RFC2617 and addressing issue4675.
"""
self.assertEqual(('user', 'ab'), urllib.splitpasswd('user:ab'))
self.assertEqual(('user', 'a\nb'), urllib.splitpasswd('user:a\nb'))
self.assertEqual(('user', 'a\tb'), urllib.splitpasswd('user:a\tb'))
self.assertEqual(('user', 'a\rb'), urllib.splitpasswd('user:a\rb'))
self.assertEqual(('user', 'a\fb'), urllib.splitpasswd('user:a\fb'))
self.assertEqual(('user', 'a\vb'), urllib.splitpasswd('user:a\vb'))
self.assertEqual(('user', 'a:b'), urllib.splitpasswd('user:a:b'))
self.assertEqual(('user', 'a b'), urllib.splitpasswd('user:a b'))
self.assertEqual(('user 2', 'ab'), urllib.splitpasswd('user 2:ab'))
self.assertEqual(('user+1', 'a+b'), urllib.splitpasswd('user+1:a+b'))
def test_splitpasswd(self):
"""Some of the password examples are not sensible, but it is added to
confirming to RFC2617 and addressing issue4675.
"""
self.assertEqual(("user", "ab"), urllib.splitpasswd("user:ab"))
self.assertEqual(("user", "a\nb"), urllib.splitpasswd("user:a\nb"))
self.assertEqual(("user", "a\tb"), urllib.splitpasswd("user:a\tb"))
self.assertEqual(("user", "a\rb"), urllib.splitpasswd("user:a\rb"))
self.assertEqual(("user", "a\fb"), urllib.splitpasswd("user:a\fb"))
self.assertEqual(("user", "a\vb"), urllib.splitpasswd("user:a\vb"))
self.assertEqual(("user", "a:b"), urllib.splitpasswd("user:a:b"))
self.assertEqual(("user", "a b"), urllib.splitpasswd("user:a b"))
self.assertEqual(("user 2", "ab"), urllib.splitpasswd("user 2:ab"))
self.assertEqual(("user+1", "a+b"), urllib.splitpasswd("user+1:a+b"))
def test_splitpasswd(self):
"""Some of the password examples are not sensible, but it is added to
confirming to RFC2617 and addressing issue4675.
"""
self.assertEqual(('user', 'ab'),urllib.splitpasswd('user:ab'))
self.assertEqual(('user', 'a\nb'),urllib.splitpasswd('user:a\nb'))
self.assertEqual(('user', 'a\tb'),urllib.splitpasswd('user:a\tb'))
self.assertEqual(('user', 'a\rb'),urllib.splitpasswd('user:a\rb'))
self.assertEqual(('user', 'a\fb'),urllib.splitpasswd('user:a\fb'))
self.assertEqual(('user', 'a\vb'),urllib.splitpasswd('user:a\vb'))
self.assertEqual(('user', 'a:b'),urllib.splitpasswd('user:a:b'))
self.assertEqual(('user', 'a b'),urllib.splitpasswd('user:a b'))
self.assertEqual(('user 2', 'ab'),urllib.splitpasswd('user 2:ab'))
self.assertEqual(('user+1', 'a+b'),urllib.splitpasswd('user+1:a+b'))
def signon(self, url='localhost:8889', login=None, password=None, startQueue=False, verbose=False):
"""Sign on to RTS2 JSON server. url - JSON API URL (can include username and login)"""
# try to get username (and password) from url
purl = urlparse.urlsplit(url)
userpass,host = urllib.splituser(purl.netloc)
(userpass, netloc) = urllib.splituser(purl.netloc)
if userpass is not None:
(login, password) = urllib.splitpasswd(userpass)
url = netloc + purl.path
if purl.query:
url += '?' + query
if purl.fragment:
url += '#' + fragment
if login is None:
self.verbose.set_active(verbose)
self.dialog.show_all()
if self.run():
self.dialog.hide()
sys.exit(-1)
self.dialog.hide()
else:
# or just create server..
createJsonServer(url, login, password, verbose=verbose)
getProxy().loadJson('/api/devices')
if startQueue:
getProxy().startQueue()
getProxy().refresh()
def __init__(self, url):
""" Initialize the downloader with the specified url string """
# FIXME: support HTTPS
scheme, host, path, params, query, fragment = urlparse.urlparse(url)
auth, host = urllib.splituser(host)
self.host, self.port = urllib.splitport(host)
if not self.port:
self.port = 80
self.username = self.password = None
if auth:
self.username, self.password = urllib.splitpasswd(auth)
self.url = urlparse.urlunparse(
(scheme, host, path, params, query, fragment))
self.nzbFilename = os.path.basename(path)
self.tempFilename = os.path.join(
Hellanzb.TEMP_DIR,
tempFilename(self.TEMP_FILENAME_PREFIX) + '.nzb')
# The HTTPDownloader
self.downloader = None
# The NZB category (e.g. 'Apps')
self.nzbCategory = None
# Whether or not the NZB file data is gzipped
self.isGzipped = False
def signon(self,
url='localhost:8889',
login=None,
password=None,
startQueue=False,
verbose=False):
"""Sign on to RTS2 JSON server. url - JSON API URL (can include username and login)"""
# try to get username (and password) from url
purl = urlparse.urlsplit(url)
userpass, host = urllib.splituser(purl.netloc)
(userpass, netloc) = urllib.splituser(purl.netloc)
if userpass is not None:
(login, password) = urllib.splitpasswd(userpass)
url = netloc + purl.path
if purl.query:
url += '?' + query
if purl.fragment:
url += '#' + fragment
if login is None:
self.verbose.set_active(verbose)
self.dialog.show_all()
if self.run():
self.dialog.hide()
sys.exit(-1)
self.dialog.hide()
else:
# or just create server..
createJsonServer(url, login, password, verbose=verbose)
getProxy().loadJson('/api/devices')
if startQueue:
getProxy().startQueue()
getProxy().refresh()
def __init__(self, url):
""" Initialize the downloader with the specified url string """
# FIXME: support HTTPS
scheme, host, path, params, query, fragment = urlparse.urlparse(url)
auth, host = urllib.splituser(host)
self.host, self.port = urllib.splitport(host)
if not self.port:
self.port = 80
self.username = self.password = None
if auth:
self.username, self.password = urllib.splitpasswd(auth)
self.url = urlparse.urlunparse((scheme, host, path, params, query, fragment))
self.nzbFilename = os.path.basename(path)
self.tempFilename = os.path.join(Hellanzb.TEMP_DIR,
tempFilename(self.TEMP_FILENAME_PREFIX) + '.nzb')
# The HTTPDownloader
self.downloader = None
# The NZB category (e.g. 'Apps')
self.nzbCategory = None
# Whether or not the NZB file data is gzipped
self.isGzipped = False
def ftp_open(self, req):
import ftplib
import mimetypes
host = req.get_host()
if not host:
raise IOError, ('ftp error', 'no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise URLError(msg)
def create_auth_baton(url):
"""Create an authentication baton for the specified URL.
:param url: URL to create auth baton for.
"""
import urlparse
(scheme, netloc, path, _, _) = urlparse.urlsplit(url)
(creds, host) = urllib.splituser(netloc)
(host, port) = urllib.splitport(host)
auth_config = SubversionAuthenticationConfig(scheme, host, port, path)
# Specify Subversion providers first, because they use file data
# rather than prompting the user.
providers = []
providers += get_stock_svn_providers()
providers += auth_config.get_svn_auth_providers()
providers += [get_ssl_client_cert_pw_provider(1)]
auth_baton = ra.Auth(providers)
if creds is not None:
(user, password) = urllib.splitpasswd(creds)
if user is not None:
auth_baton.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME, user)
if password is not None:
auth_baton.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_PASSWORD, password)
return auth_baton
def __init__(self, uri, cookiejar=None, use_datetime=0):
Transport.__init__(self, use_datetime=use_datetime)
self.opener = build_opener()
# Parse auth (user:passwd) from the uri
urltype, rest = splittype(uri)
host, rest = splithost(rest)
auth, host = splituser(host)
self.uri = urltype + '://' + host + rest
# Handle HTTP Basic authentication
if auth is not None:
user, passwd = splitpasswd(auth)
passwdmgr = HTTPPasswordMgrWithDefaultRealm()
passwdmgr.add_password(realm=None,
uri=self.uri,
user=user,
passwd=passwd)
authhandler = HTTPBasicAuthHandler(passwdmgr)
self.opener.add_handler(authhandler)
# Handle HTTP Cookies
if cookiejar is not None:
self.opener.add_handler(HTTPCookieProcessor(cookiejar))
def parse_address_info(cls, server_addr="nats://*****:*****@127.0.0.1:4222"):
'''\
parse the metadata nats server uri;
Params:
=====
addr: nats server address;
Returns:
=====
user: username to login nats server;
pswd: password to login nats server;
host: ip address of nats server;
port: port of nats server
'''
if type(server_addr) is not str:
raise NotImplementException
protocol, after_split = urllib.splittype(server_addr)
if not protocol == "nats":
raise NotImplementException
auth_len = len(server_addr.split('@'))
if auth_len > 1:
auth, after_split = urllib.splituser(after_split)
user_raw, pswd = urllib.splitpasswd(auth)
user = user_raw.lstrip("/")
_, after_split = urllib.splithost(after_split)
host, port = urllib.splitport(after_split)
else:
user = pswd = None
host, port = urllib.splitport(after_split)
return user, pswd, host, int(port)
def ftp_open(self, req):
import ftplib
import mimetypes
host = req.get_host()
if not host:
raise URLError('ftp error: no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise URLError(msg)
def __init__(self, baseUri, headers=None, maxClients=None,
maxConnections=None):
self._headers = headers or HTTPHeaders()
self._user = None
self._passwd = None
baseUri = baseUri.rstrip('/')
self._scheme, loc, self._path, query, frag = urlparse.urlsplit(baseUri)
userpass, self._hostport = urllib.splituser(loc)
if userpass:
self._user, self._passwd = urllib.splitpasswd(userpass)
self._baseUri = urlparse.urlunsplit((self._scheme, self._hostport,
self._path, None, None))
if self._scheme not in ('http', 'https'):
raise ValueError(self._scheme)
self._dispatcher = RequestDispatcher(maxClients=maxClients,
maxConnections=maxConnections)
self._queryFragment = urlparse.urlunsplit(('', '', '', query, frag))
def __init__(self, url):
self.scheme, self.netloc, self.path, query, self.fragment = \
urlparse.urlsplit(url)
self.user, self.host = urllib.splituser(self.netloc)
if not self.user:
try: # get userid from current user
self.user = pwd.getpwuid(os.getuid())[0]
except KeyError:
self.user = "******"
self.user = self.user or ""
self.userid = self.username = self.user # aliases
self.adapter = self.scheme
self.host, self.port = urllib.splitport(self.host)
self.user, self.password = urllib.splitpasswd(self.user or "")
self.password = self.password or ""
self.path = self.path[1:]
self.database = self.path
if query:
try:
self.query = dict([
(k, urllib.unquote_plus(v)) for k, v in
[e.split("=", 1) for e in self.query.split("&", 1)]
])
except ValueError, e:
raise ValueError, "Query part in url cannot be parsed: %r (%s)" % (
self.query, e)
def parse_address_info(cls, server_addr="nats://*****:*****@127.0.0.1:4222"):
'''\
parse the metadata nats server uri;
Params:
=====
addr: nats server address;
Returns:
=====
user: username to login nats server;
pswd: password to login nats server;
host: ip address of nats server;
port: port of nats server
'''
if type(server_addr) is not str:
raise NotImplementException
protocol, after_split = urllib.splittype(server_addr)
if not protocol == "nats":
raise NotImplementException
auth_len = len(server_addr.split('@'))
if auth_len > 1:
auth, after_split = urllib.splituser(after_split)
user_raw, pswd = urllib.splitpasswd(auth)
user = user_raw.lstrip("/")
_, after_split = urllib.splithost(after_split)
host, port = urllib.splitport(after_split)
else:
user = pswd = None
host, port = urllib.splitport(after_split)
return user, pswd, host, int(port)
def get_user_password (self):
"""Get tuple (user, password) from configured authentication.
Both user and password can be None.
"""
if self.userinfo:
# URL itself has authentication info
return urllib.splitpasswd(self.userinfo)
return self.aggregate.config.get_user_password(self.url)
def get_user_password(self):
"""Get tuple (user, password) from configured authentication.
Both user and password can be None.
"""
if self.userinfo:
# URL itself has authentication info
return urllib.splitpasswd(self.userinfo)
return self.aggregate.config.get_user_password(self.url)
def __init__(self, uri, username=None, password=None, *args, **kwargs):
xmlrpclib.ServerProxy.__init__(self, uri, *args, **kwargs)
# Hide password
userpass, hostport = urllib.splituser(self.__host)
if userpass and not username:
self.__host = hostport
username, password = urllib.splitpasswd(userpass)
if username:
password = util.ProtectedString(urllib.quote(password))
self.__host = ProtectedTemplate('${user}:${password}@${host}',
user=username, password=password, host=self.__host)
def ftp_open(self, req):
import ftplib
import mimetypes
host = req.get_host()
if not host:
raise URLError('ftp error: no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
passwd = passwd or ''
try:
host = socket.gethostbyname(host)
except socket.error as msg:
raise URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitvalue(attr)
if attr.lower() == 'type' and value in ('a', 'A', 'i', 'I',
'd', 'D'):
type = value.upper()
fp, retrlen = fw.retrfile(file, type)
headers = ''
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += 'Content-type: %s\n' % mtype
if retrlen is not None and retrlen >= 0:
headers += 'Content-length: %d\n' % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors as msg:
raise URLError, 'ftp error: %s' % msg, sys.exc_info()[2]
return
def ftp_open(self, req):
import ftplib
import mimetypes
host = req.get_host()
if not host:
raise URLError('ftp error: no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
passwd = passwd or ''
try:
host = socket.gethostbyname(host)
except socket.error as msg:
raise URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitvalue(attr)
if attr.lower() == 'type' and value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
fp, retrlen = fw.retrfile(file, type)
headers = ''
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += 'Content-type: %s\n' % mtype
if retrlen is not None and retrlen >= 0:
headers += 'Content-length: %d\n' % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors as msg:
raise URLError, 'ftp error: %s' % msg, sys.exc_info()[2]
return
def parse_backend(url):
def unescape(s):
return urllib.unquote(s.replace('+', ' '))
try:
proto, u = urllib.splittype(url)
auth, path = urllib.splithost(u)
user, host = urllib.splituser(auth)
user, password = urllib.splitpasswd(user)
host, _ = urllib.splitport(host)
_, query = urllib.splitquery(path)
args = query.split('&')
hot_user = None
hot_pass = None
level = 0
mode = -1
for arg in args:
k, v = arg.split('=', 2)
v = unescape(v)
if k == 'hot_user':
hot_user = v
elif k == 'hot_pass':
hot_pass = v
elif k == 'level':
level = parse_level(v)
elif k == 'mode':
mode = parse_mode(v)
elif k == 'extra':
hot_addr, extra = parse_extra(v)
if hot_user != None and hot_addr == None:
hot_addr = host
Backend = namedtuple('Backend', [
'address', 'user', 'password', 'hot_addr', 'hot_user', 'hot_pass',
'extra', 'level', 'mode'
])
return Backend(address=host,
user=user,
password=password,
hot_addr=hot_addr,
hot_user=hot_user,
hot_pass=hot_pass,
extra=extra,
level=level,
mode=mode)
except Exception:
logging.getLogger('mwtm_executor').error('bad backend: %s',
url,
exc_info=True)
_exit(1)
def __init__(self, uri, username=None, password=None, *args, **kwargs):
xmlrpclib.ServerProxy.__init__(self, uri, *args, **kwargs)
# Hide password
userpass, hostport = urllib.splituser(self.__host)
if userpass and not username:
self.__host = hostport
username, password = urllib.splitpasswd(userpass)
if username:
password = util.ProtectedString(urllib.quote(password))
self.__host = ProtectedTemplate('${user}:${password}@${host}',
user=username,
password=password,
host=self.__host)
def ParseUrl(value):
type = ""
host = ""
port = ""
path = ""
username = ""
password = ""
type, value = splittype(value or "")
value, path = splithost(value or "")
usps, value = splituser(value or "")
if usps:
username, password = splitpasswd(usps)
host, port = splitnport(value or "", None)
return type or "", host or "", str(
port) if port else "", path or "", username or "", password or ""
def check_args(options, args):
if not args:
return
# someway to test if it is a SIP URI
if 'sip:' in args[0]:
setattr(options, 'target_uri', args[0])
userpass, hostport = urllib.splituser(args[0])
if hostport[:4] == 'sip:': hostport = hostport[4:]
host, port = urllib.splitnport(hostport, 5060)
# XXX Overwrites!
setattr(options, 'target_addr', (host, port))
if userpass:
user, passwd = urllib.splitpasswd(userpass)
user and setattr(options, 'user', user)
passwd and setattr(options, 'passwd', passwd)
def check_args(options, args):
if not args:
return
# someway to test if it is a SIP URI
if 'sip:' in args[0]:
setattr(options, 'target_uri', args[0])
userpass, hostport = urllib.splituser(args[0])
if hostport[:4] == 'sip:': hostport = hostport[4:]
host, port = urllib.splitnport(hostport, 5060)
# XXX Overwrites!
setattr(options, 'target_addr', (host, port))
if userpass:
user, passwd = urllib.splitpasswd(userpass)
user and setattr(options, 'user', user)
passwd and setattr(options, 'passwd', passwd)
def _getconnection(netloc):
"""Return a FTP connection object to the specified server."""
# NOTE: this method is not thread safe
if _ftpconnections.has_key(netloc):
return _ftpconnections[netloc]
# split url into useful parts
(userpass, host) = urllib.splituser(netloc)
if userpass is not None:
(user, passwd) = urllib.splitpasswd(userpass)
else:
(user, passwd) = ('anonymous', '')
(host, port) = urllib.splitnport(host, ftplib.FTP_PORT)
# initialize a new connection
ftp = ftplib.FTP()
debugio.debug('schemes.ftp._getconnection(): CONNECT: '+ftp.connect(host, port))
debugio.debug('schemes.ftp._getconnection(): LOGIN: '+ftp.login(user, passwd))
_ftpconnections[netloc] = ftp
return ftp
def _parse_proxy(proxy):
scheme, r_scheme = splittype(proxy)
if not r_scheme.startswith('/'):
scheme = None
authority = proxy
else:
if not r_scheme.startswith('//'):
raise ValueError('proxy URL with no authority: %r' % proxy)
end = r_scheme.find('/', 2)
if end == -1:
end = None
authority = r_scheme[2:end]
userinfo, hostport = splituser(authority)
if userinfo is not None:
user, password = splitpasswd(userinfo)
else:
user = password = None
return (scheme, user, password, hostport)
def urlSplit(url, defaultPort = None):
"""A function to split a URL in the format
<scheme>://<user>:<pass>@<host>:<port>/<path>;<params>#<fragment>
into a tuple
(<scheme>, <user>, <pass>, <host>, <port>, <path>, <params>, <fragment>)
Any missing pieces (user/pass) will be set to None.
If the port is missing, it will be set to defaultPort; otherwise, the port
should be a numeric value.
"""
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
userpass, hostport = urllib.splituser(netloc)
host, port = urllib.splitnport(hostport, None)
if userpass:
user, passwd = urllib.splitpasswd(userpass)
else:
user, passwd = None, None
return scheme, user, passwd, host, port, path, \
query or None, fragment or None
def urlSplit(url, defaultPort=None):
"""A function to split a URL in the format
<scheme>://<user>:<pass>@<host>:<port>/<path>;<params>#<fragment>
into a tuple
(<scheme>, <user>, <pass>, <host>, <port>, <path>, <params>, <fragment>)
Any missing pieces (user/pass) will be set to None.
If the port is missing, it will be set to defaultPort; otherwise, the port
should be a numeric value.
"""
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
userpass, hostport = urllib.splituser(netloc)
host, port = urllib.splitnport(hostport, None)
if userpass:
user, passwd = urllib.splitpasswd(userpass)
else:
user, passwd = None, None
return scheme, user, passwd, host, port, path, \
query or None, fragment or None
def __init__(self, url):
type, uri = urllib.splittype(url)
#if type not in ("http", "https"):
# raise IOError, "unsupported XML-RPC protocol"
self.host, self.url = urllib.splithost(uri)
if self.url == "":
self.url = "/"
self.user = self.password = None
self.user, self.host = urllib.splituser(self.host)
try:
self.user, self.password = urllib.splitpasswd(self.user)
except TypeError:
pass
self.host, self.port = urllib.splitport(self.host)
self.port = int(self.port)
self.secure = type == 'https'
def __init__(self, url):
type, uri = urllib.splittype(url)
#if type not in ("http", "https"):
# raise IOError, "unsupported XML-RPC protocol"
self.host, self.url = urllib.splithost(uri)
if self.url == "":
self.url = "/"
self.user = self.password = None
self.user, self.host = urllib.splituser(self.host)
try:
self.user, self.password = urllib.splitpasswd(self.user)
except TypeError:
pass
self.host, self.port = urllib.splitport(self.host)
self.port = int(self.port)
self.secure = type == 'https'
def build_http_request(self, host, handler, request_body):
token = None
user = None
auth, host = urllib.splituser(host)
if auth:
user, token = urllib.splitpasswd(auth)
url = self._scheme + "://" + host + handler
if user is not None and token is None:
token = self.auth_backend.get_token_for_endpoint(user, url)
if token is None:
raise LavaCommandError(
"Username provided but no token found.")
request = urllib2.Request(url, request_body)
request.add_header("Content-Type", "text/xml")
if token:
auth = base64.b64encode(urllib.unquote(user + ':' + token))
request.add_header("Authorization", "Basic " + auth)
return request
def parse_remote_repository (uri, ) :
_pa = urlparse.urlsplit(uri)
(_a_user, _a_host, ) = urllib.splituser(_pa.netloc, )
_user, _password = None, None
if _a_user :
(_user, _password, ) = urllib.splitpasswd(_a_user, )
_defaut_port = _pa.scheme.lower() in ("svn+ssh", "ssh", ) and 22 or None
(_host, _port, ) = urllib.splitnport(_a_host, defport=_defaut_port, )
return dict(
host=_host,
port=_port,
scheme=_pa.scheme.lower(),
user=_user,
password=_password,
path=os.path.normpath(
_pa.path.strip().startswith("/") and _pa.path or ("/" + _pa.path)
),
)
def parse_backend(url):
def unescape(s):
return urllib.unquote(s.replace('+', ' '))
try:
proto, u = urllib.splittype(url)
auth, path = urllib.splithost(u)
user, host = urllib.splituser(auth)
user, password = urllib.splitpasswd(user)
host, _ = urllib.splitport(host)
_, query = urllib.splitquery(path)
args = query.split('&')
hot_user = None
hot_pass = None
level = 0
mode = -1
for arg in args:
k, v = arg.split('=', 2)
v = unescape(v)
if k == 'hot_user':
hot_user = v
elif k == 'hot_pass':
hot_pass = v
elif k == 'level':
level = parse_level(v)
elif k == 'mode':
mode = parse_mode(v)
elif k == 'extra':
hot_addr, extra = parse_extra(v)
if hot_user != None and hot_addr == None:
hot_addr = host
Backend = namedtuple('Backend', ['address', 'user', 'password',
'hot_addr', 'hot_user', 'hot_pass',
'extra', 'level', 'mode'])
return Backend(address=host, user=user, password=password,
hot_addr=hot_addr, hot_user=hot_user, hot_pass=hot_pass,
extra=extra, level=level, mode=mode)
except Exception:
logging.getLogger('mwtm_executor').error('bad backend: %s', url,
exc_info=True)
_exit(1)
def parse_uri(uri, **kwargs):
scheme, rest = urllib.splittype(uri)
host, rest = urllib.splithost(rest)
user, host = urllib.splituser(host)
if user:
username, password = urllib.splitpasswd(user)
else:
username = password = None
host, port = urllib.splitnport(host)
path, query = urllib.splitquery(rest)
if query:
kwargs.update(dict(cgi.parse_qsl(query)))
return dict(
scheme=scheme,
host=host,
username=username,
password=password,
port=port,
path=path,
query=kwargs)
def _getBaseServerUrlData(self):
"""
Fetch serverUrl from ~/.rbuilderrc if it exists and is specified;
removes user and password from the URL and returns them separately.
@return serverUrl, user, password
"""
serverUrl = self._getBaseServerUrl()
if not serverUrl:
return (None, None, None)
scheme, rest = serverUrl.split(':', 1)
host = urllib.splithost(rest)[0]
user = urllib.splituser(host)[0]
if user:
user, password = urllib.splitpasswd(user)
else:
password = None
if password:
serverUrl = serverUrl.replace(':%s' % password, '', 1)
if user:
serverUrl = serverUrl.replace('%s@' % user, '', 1)
return serverUrl, user, password
def _getBaseServerUrlData(self):
"""
Fetch serverUrl from ~/.rbuilderrc if it exists and is specified;
removes user and password from the URL and returns them separately.
@return serverUrl, user, password
"""
serverUrl = self._getBaseServerUrl()
if not serverUrl:
return (None, None, None)
scheme, rest = serverUrl.split(":", 1)
host = urllib.splithost(rest)[0]
user = urllib.splituser(host)[0]
if user:
user, password = urllib.splitpasswd(user)
else:
password = None
if password:
serverUrl = serverUrl.replace(":%s" % password, "", 1)
if user:
serverUrl = serverUrl.replace("%s@" % user, "", 1)
return serverUrl, user, password
def _parse_proxy(proxy):
scheme, r_scheme = splittype(proxy)
if not r_scheme.startswith('/'):
scheme = None
authority = proxy
else:
if not r_scheme.startswith('//'):
raise ValueError('proxy URL with no authority: %r' % proxy)
end = r_scheme.find('/', 2)
if end == -1:
end = None
authority = r_scheme[2:end]
userinfo, hostport = splituser(authority)
if userinfo is not None:
user, password = splitpasswd(userinfo)
else:
user = password = None
return (scheme,
user,
password,
hostport)
def _parseURI(uri):
protocol, request = urllib.splittype(uri)
user, password, port = None, None, None
host, path = urllib.splithost(request)
if host:
# Python < 2.7 have a problem - splituser() calls unquote() too early
#user, host = urllib.splituser(host)
if '@' in host:
user, host = host.split('@', 1)
if user:
user, password = [
x and urllib.unquote(x) or None
for x in urllib.splitpasswd(user)
]
host, port = urllib.splitport(host)
if port: port = int(port)
elif host == '':
host = None
# hash-tag is splitted but ignored
path, tag = urllib.splittag(path)
path, query = urllib.splitquery(path)
path = urllib.unquote(path)
if (os.name == 'nt') and (len(path) > 2):
# Preserve backward compatibility with URIs like /C|/path;
# replace '|' by ':'
if path[2] == '|':
path = "%s:%s" % (path[0:2], path[3:])
# Remove leading slash
if (path[0] == '/') and (path[2] == ':'):
path = path[1:]
args = {}
if query:
for name, value in parse_qsl(query):
args[name] = value
return user, password, host, port, path, args
def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError, ("ftp error", "no host given")
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or "")
passwd = unquote(passwd or "")
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise URLError(msg)
def parse_url(url, default_port=None):
'''
Parse url in the following form:
PROTO://[USER:[:PASSWD]@]HOST[:PORT][/PATH[;ATTR][?QUERY]]
A tuple containing (proto, user, passwd, host, port, path, tag, attrs, query) is returned,
where `attrs' is a tuple containing ('attr1=value1', 'attr2=value2', ...)
'''
proto, user, passwd, host, port, path, tag, attrs, query = (None, ) * 9
try:
proto, tmp_host = urllib.splittype(url)
tmp_host, tmp_path = urllib.splithost(tmp_host)
tmp_user, tmp_host = urllib.splituser(tmp_host)
if tmp_user:
user, passwd = urllib.splitpasswd(tmp_user)
host, port = urllib.splitport(tmp_host)
port = int(port) if port else default_port
tmp_path, query = urllib.splitquery(tmp_path)
tmp_path, attrs = urllib.splitattr(tmp_path)
path, tag = urllib.splittag(tmp_path)
except Exception, err:
raise Exception('parse_db_url error - {0}'.format(str(err)))
def __init__(self, url, headers = None):
self.scheme = None
self.user = None
self.passwd = None
self.hostport = None
self.path = None
self.headers = headers or {}
urltype, url = urllib.splittype(url)
if urltype:
self.scheme = urltype.lower()
host, self.path = urllib.splithost(url)
if host:
user_passwd, self.hostport = urllib.splituser(host)
if user_passwd:
self.user, self.passwd = urllib.splitpasswd(user_passwd)
if self.scheme not in ['http', 'https']:
raise ValueError(self.scheme)
self._connection = None
def __init__(self, uri, cookiejar=None, use_datetime=0): Transport.__init__(self, use_datetime=use_datetime) self.opener = build_opener() # Parse auth (user:passwd) from the uri urltype, rest = splittype(uri) host, rest = splithost(rest) auth, host = splituser(host) self.uri = urltype + '://' + host + rest # Handle HTTP Basic authentication if auth is not None: user, passwd = splitpasswd(auth) passwdmgr = HTTPPasswordMgrWithDefaultRealm() passwdmgr.add_password(realm=None, uri=self.uri, user=user, passwd=passwd) authhandler = HTTPBasicAuthHandler(passwdmgr) self.opener.add_handler(authhandler) # Handle HTTP Cookies if cookiejar is not None: self.opener.add_handler(HTTPCookieProcessor(cookiejar))
def parse_url(url, default_port=None):
'''
Parse url in the following form:
PROTO://[USER:[:PASSWD]@]HOST[:PORT][/PATH[;ATTR][?QUERY]]
A tuple containing (proto, user, passwd, host, port, path, tag, attrs, query) is returned,
where `attrs' is a tuple containing ('attr1=value1', 'attr2=value2', ...)
'''
proto, user, passwd, host, port, path, tag, attrs, query = (None, ) * 9
try:
proto, tmp_host = urllib.splittype(url)
tmp_host, tmp_path = urllib.splithost(tmp_host)
tmp_user, tmp_host = urllib.splituser(tmp_host)
if tmp_user:
user, passwd = urllib.splitpasswd(tmp_user)
host, port = urllib.splitport(tmp_host)
port = int(port) if port else default_port
tmp_path, query = urllib.splitquery(tmp_path)
tmp_path, attrs = urllib.splitattr(tmp_path)
path, tag = urllib.splittag(tmp_path)
except Exception, err:
raise Exception('parse_db_url error - {0}'.format(str(err)))
def __init__(self, url):
self.scheme, self.netloc, self.path, query, self.fragment = \
urlparse.urlsplit(url)
self.user, self.host = urllib.splituser(self.netloc)
if not self.user:
try: # get userid from current user
self.user = pwd.getpwuid(os.getuid())[0]
except KeyError: self.user = "******"
self.user = self.user or ""
self.userid = self.username = self.user # aliases
self.adapter = self.scheme
self.host, self.port = urllib.splitport(self.host)
self.user, self.password = urllib.splitpasswd(self.user or "")
self.password = self.password or ""
self.path = self.path[1:]
self.database = self.path
if query:
try:
self.query = dict([
(k,urllib.unquote_plus(v)) for k,v in [e.split("=", 1) for e in self.query.split("&", 1)]
])
except ValueError, e:
raise ValueError, "Query part in url cannot be parsed: %r (%s)" % (self.query, e)
def parse_proxy(proxy):
""" _parse_proxy from urllib """
scheme, r_scheme = splittype(proxy)
if not r_scheme.startswith("/"):
# authority
scheme = None
authority = proxy
else:
# URL
if not r_scheme.startswith("//"):
raise ValueError("proxy URL with no authority: %r" % proxy)
# We have an authority, so for RFC 3986-compliant URLs (by ss 3.
# and 3.3.), path is empty or starts with '/'
end = r_scheme.find("/", 2)
if end == -1:
end = None
authority = r_scheme[2:end]
userinfo, hostport = splituser(authority)
if userinfo is not None:
user, password = splitpasswd(userinfo)
else:
user = password = None
return scheme, user, password, hostport
def parse_proxy(proxy):
""" _parse_proxy from urllib """
scheme, r_scheme = splittype(proxy)
if not r_scheme.startswith("/"):
# authority
scheme = None
authority = proxy
else:
# URL
if not r_scheme.startswith("//"):
raise ValueError("proxy URL with no authority: %r" % proxy)
# We have an authority, so for RFC 3986-compliant URLs (by ss 3.
# and 3.3.), path is empty or starts with '/'
end = r_scheme.find("/", 2)
if end == -1:
end = None
authority = r_scheme[2:end]
userinfo, hostport = splituser(authority)
if userinfo is not None:
user, password = splitpasswd(userinfo)
else:
user = password = None
return scheme, user, password, hostport
def _parseURI(uri):
protocol, request = urllib.splittype(uri)
user, password, port = None, None, None
host, path = urllib.splithost(request)
if host:
# Python < 2.7 have a problem - splituser() calls unquote() too early
#user, host = urllib.splituser(host)
if '@' in host:
user, host = host.split('@', 1)
if user:
user, password = [x and urllib.unquote(x) or None for x in urllib.splitpasswd(user)]
host, port = urllib.splitport(host)
if port: port = int(port)
elif host == '':
host = None
# hash-tag is splitted but ignored
path, tag = urllib.splittag(path)
path, query = urllib.splitquery(path)
path = urllib.unquote(path)
if (os.name == 'nt') and (len(path) > 2):
# Preserve backward compatibility with URIs like /C|/path;
# replace '|' by ':'
if path[2] == '|':
path = "%s:%s" % (path[0:2], path[3:])
# Remove leading slash
if (path[0] == '/') and (path[2] == ':'):
path = path[1:]
args = {}
if query:
for name, value in parse_qsl(query):
args[name] = value
return user, password, host, port, path, args
def sane_config(options):
if not options.target_uri and not options.to_uri:
options.target_uri = options.to_uri = '"siping" <sip:siping@localhost>'
elif options.target_uri and not options.to_uri:
options.to_uri = options.target_uri
elif options.to_uri and not options.target_uri:
options.target_uri = options.to_uri
#if not options.target_uri: options.target_uri = options.to_uri
if not options.from_uri: options.from_uri = options.to_uri
if not options.contacts_uri: options.contacts_uri.append( options.from_uri )
if not options.cseq_method:
options.cseq_method = options.method
if not hasattr(options, 'target_addr'):
hp = options.to_uri.find('sip:')
hp = options.to_uri[hp+4:]
if '@' in hp:
up, hp = urllib.splituser(hp)
if hp[-1] == '>': hp = hp[:-1]
h, p = urllib.splitnport(hp, 5060)
if type(p) != int: p = int(p)
setattr(options, 'target_addr', (h, p))
if up:
u, p = urllib.splitpasswd(up)
setattr(options, 'username', u)
setattr(options, 'password', p)
if not options.target_addr[1]: # no port
options.target_addr = (options.target_addr[0], 5060)
def parse (url):
import urllib
import socket
db_conf = {};
start_idx = url.find ('mysql://');
if start_idx != 0:
raise Exception ("bad mysql format (%s)" %url);
try:
start_idx = len ('mysql://');
end_idx = url.find ('/', start_idx);
if end_idx >= 0:
tidx = url.find ('/', end_idx+1);
if tidx < 0:
db_conf['db'] = url[end_idx+1:];
db_conf['tbl'] = None;
else:
db_conf['db'] = url[end_idx+1:tidx];
db_conf['tbl'] = url[tidx+1:]
else:
db_conf['db'] = None;
db_conf['tbl'] = None;
passwd, port = urllib.splituser (url[start_idx:end_idx]);
db_conf['user'], db_conf['pass'] = urllib.splitpasswd (passwd);
host, db_port = urllib.splitport (port);
#try:
# socket.inet_aton (host)
#except socket.error:
# raise
db_conf['host'] = host
if db_port is not None:
db_conf['port'] = int (db_port);
else:
db_conf['port'] = 3306
except Exception, msg:
raise Exception ("bad mysql format (%s), %s" % (url, msg));
def sane_config(options):
if not options.target_uri and not options.to_uri:
options.target_uri = options.to_uri = '"siping" <sip:siping@localhost>'
elif options.target_uri and not options.to_uri:
options.to_uri = options.target_uri
elif options.to_uri and not options.target_uri:
options.target_uri = options.to_uri
#if not options.target_uri: options.target_uri = options.to_uri
if not options.from_uri: options.from_uri = options.to_uri
if not options.contacts_uri: options.contacts_uri.append(options.from_uri)
if not options.cseq_method:
options.cseq_method = options.method
if not hasattr(options, 'target_addr'):
hp = options.to_uri.find('sip:')
hp = options.to_uri[hp + 4:]
if '@' in hp:
up, hp = urllib.splituser(hp)
if hp[-1] == '>': hp = hp[:-1]
h, p = urllib.splitnport(hp, 5060)
if type(p) != int: p = int(p)
setattr(options, 'target_addr', (h, p))
if up:
u, p = urllib.splitpasswd(up)
setattr(options, 'username', u)
setattr(options, 'password', p)
if not options.target_addr[1]: # no port
options.target_addr = (options.target_addr[0], 5060)
def parse(url):
import urllib
import socket
db_conf = {}
start_idx = url.find('mysql://')
if start_idx != 0:
raise Exception("bad mysql format (%s)" % url)
try:
start_idx = len('mysql://')
end_idx = url.find('/', start_idx)
if end_idx >= 0:
tidx = url.find('/', end_idx + 1)
if tidx < 0:
db_conf['db'] = url[end_idx + 1:]
db_conf['tbl'] = None
else:
db_conf['db'] = url[end_idx + 1:tidx]
db_conf['tbl'] = url[tidx + 1:]
else:
db_conf['db'] = None
db_conf['tbl'] = None
passwd, port = urllib.splituser(url[start_idx:end_idx])
db_conf['user'], db_conf['pass'] = urllib.splitpasswd(passwd)
host, db_port = urllib.splitport(port)
#try:
# socket.inet_aton (host)
#except socket.error:
# raise
db_conf['host'] = host
if db_port is not None:
db_conf['port'] = int(db_port)
else:
db_conf['port'] = 3306
except Exception, msg:
raise Exception("bad mysql format (%s), %s" % (url, msg))
def _parse_proxy(proxy):
"""Return (scheme, user, password, host/port) given a URL or an authority.
If a URL is supplied, it must have an authority (host:port) component.
According to RFC 3986, having an authority component means the URL must
have two slashes after the scheme:
>>> _parse_proxy('file:/ftp.example.com/')
Traceback (most recent call last):
ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
The first three items of the returned tuple may be None.
Examples of authority parsing:
>>> _parse_proxy('proxy.example.com')
(None, None, None, 'proxy.example.com')
>>> _parse_proxy('proxy.example.com:3128')
(None, None, None, 'proxy.example.com:3128')
The authority component may optionally include userinfo (assumed to be
username:password):
>>> _parse_proxy('joe:[email protected]')
(None, 'joe', 'password', 'proxy.example.com')
>>> _parse_proxy('joe:[email protected]:3128')
(None, 'joe', 'password', 'proxy.example.com:3128')
Same examples, but with URLs instead:
>>> _parse_proxy('http://proxy.example.com/')
('http', None, None, 'proxy.example.com')
>>> _parse_proxy('http://proxy.example.com:3128/')
('http', None, None, 'proxy.example.com:3128')
>>> _parse_proxy('http://*****:*****@proxy.example.com/')
('http', 'joe', 'password', 'proxy.example.com')
>>> _parse_proxy('http://*****:*****@proxy.example.com:3128')
('http', 'joe', 'password', 'proxy.example.com:3128')
Everything after the authority is ignored:
>>> _parse_proxy('ftp://*****:*****@proxy.example.com/rubbish:3128')
('ftp', 'joe', 'password', 'proxy.example.com')
Test for no trailing '/' case:
>>> _parse_proxy('http://*****:*****@proxy.example.com')
('http', 'joe', 'password', 'proxy.example.com')
"""
scheme, r_scheme = splittype(proxy)
if not r_scheme.startswith("/"):
# authority
scheme = None
authority = proxy
else:
# URL
if not r_scheme.startswith("//"):
raise ValueError("proxy URL with no authority: %r" % proxy)
# We have an authority, so for RFC 3986-compliant URLs (by ss 3.
# and 3.3.), path is empty or starts with '/'
end = r_scheme.find("/", 2)
if end == -1:
end = None
authority = r_scheme[2:end]
userinfo, hostport = splituser(authority)
if userinfo is not None:
user, password = splitpasswd(userinfo)
else:
user = password = None
return scheme, user, password, hostport
def get_repository(self, repos_type, repos_dir, authname):
assert repos_type == 'perforce'
import urllib
urltype, url = urllib.splittype(repos_dir)
assert urltype == 'p4' or url == 'p4'
options = dict(self.config.options('perforce'))
if urltype != None:
machine, path_query = urllib.splithost(url)
user_passwd, host_port = urllib.splituser(machine)
user, password = urllib.splitpasswd(user_passwd)
self._update_option(options, 'port', host_port)
self._update_option(options, 'password', password)
self._update_option(options, 'user', user)
path, query = urllib.splitquery(path_query)
if path and path != '':
for attr in self._splitattributes(query):
key, val = urllib.splitvalue(attr)
self._update_option(options, key, val)
self._update_option(options, 'path', path)
self.log.debug("get_repository options : %s" % (options))
if 'port' not in options:
raise TracError(
message="Missing 'port' value in [perforce] config section.",
title="TracPerforce configuration error",
)
# Try to connect to the Perforce server
from perforce import Connection, ConnectionFailed
p4 = Connection(
port=options['port'],
api='58', # Limit to 2005.2 behaviour
)
try:
from trac import __version__ as tracVersion
p4.connect(prog='Trac', version=tracVersion)
except ConnectionFailed:
raise TracError(
message="Could not connect to Perforce repository.",
title="Perforce connection error",
)
if 'user' not in options:
raise TracError(
message="Missing 'user' value in [perforce] config section.",
title="Perforce configuration error",
)
p4.user = options['user']
if 'password' in options:
p4.password = options['password']
else:
p4.password = ''
if 'unicode' in options:
if options['unicode'] == '1':
p4.charset = 'utf8'
elif options['unicode'] == '0':
p4.charset = 'none'
else:
raise TracError(
message="Invalid 'unicode' value in [perforce] config " \
"section.",
title="Perforce configuration error",
)
else:
p4.charset = 'none'
if 'language' in options:
p4.language = options['language']
else:
p4.language = ''
jobPrefixLength = 3 # default value because by default prefix is 'job'
if 'job_prefix' in options:
jobPrefixLength = len(options['job_prefix'])
p4.client = ''
repos = PerforceRepository(p4, None, self.log, jobPrefixLength)
crepos = CachedRepository(self.env.get_db_cnx(), repos, None, self.log)
return crepos
def __init__(self, url, method, params):
Assert(method == 'GET')
netloc, path = splithost(url)
if not netloc: raise IOError, ('ftp error', 'no host given')
host, port = splitport(netloc)
user, host = splituser(host)
if user: user, passwd = splitpasswd(user)
else: passwd = None
host = socket.gethostbyname(host)
if port:
try:
port = string.atoi(port)
except string.atoi_error:
raise IOError, ('ftp error', 'bad port')
else:
port = ftplib.FTP_PORT
path, attrs = splitattr(path)
self.url = "ftp://%s%s" % (netloc, path)
dirs = string.splitfields(path, '/')
dirs, file = dirs[:-1], dirs[-1]
self.content_length = None
if not file:
self.content_type, self.content_encoding = None, None
type = 'd'
else:
self.content_type, self.content_encoding = app.guess_type(file)
if self.content_encoding:
type = 'i'
elif self.content_type and self.content_type[:5] == 'text/':
type = 'a'
elif file[-1] == '/':
type = 'd'
else:
type = 'i'
if dirs and not dirs[0]: dirs = dirs[1:]
key = (user, host, port, string.joinfields(dirs, '/'))
self.debuglevel = None
try:
if not ftpcache.has_key(key):
ftpcache[key] = []
for attr in attrs:
[attr, value] = map(string.lower, splitvalue(attr))
if attr == 'type' and value in ('a', 'i', 'd'):
type = value
elif attr == 'debug':
try:
self.debuglevel = string.atoi(value)
except string.atoi_error:
pass
candidates = ftpcache[key]
for cand in candidates:
if not cand.busy():
break
else:
cand = ftpwrapper(user, passwd,
host, port, dirs, self.debuglevel)
candidates.append(cand)
# XXX Ought to clean the cache every once in a while
self.cand = cand
self.sock, self.isdir = cand.retrfile(file, type)
self.content_length = cand.content_length
except ftplib.all_errors, msg:
raise IOError, ('ftp error', msg)
def downloadFileUsingURLLib(url,
workingdir=os.getcwd(),
dirPrefix=None,
verbose=False,
timeout=60,
destFile=None):
global logger
global passwordManager
global currentDownloadRetryCount
import errno
if currentDownloadRetryCount <= 0:
sys.stderr.write(
'Error: reaching maximum retry for downloading.\nExiting.\n')
logger.log(logging.DEBUG, 'program exists with error code 1\n')
sys.exit(1)
else:
currentDownloadRetryCount = currentDownloadRetryCount - 1
user = getUserFromPasswordManager(logger)
username = options.username
if (user != None):
username = user.getUsername()
cookiesFilePath = getCookiesFilePath(url, username)
logger.log(logging.DEBUG, 'Starting downloading file using urllib2')
logger.log(logging.DEBUG, 'url=' + url)
logger.log(logging.DEBUG, 'workingdir=' + workingdir)
if dirPrefix != None:
logger.log(logging.DEBUG, 'dirPrefix=' + dirPrefix)
try:
cj = cookielib.MozillaCookieJar()
if os.path.exists(cookiesFilePath):
cj.load(cookiesFilePath)
except cookielib.LoadError:
cj = cookielib.LWPCookieJar()
try:
if os.path.exists(cookiesFilePath):
cj.load(cookiesFilePath)
except cookielib.LoadError:
pass
opener = None
scheme, sel_url = urllib.splittype(url)
if scheme.lower() == 'ftp':
import re
valid_ftp_url = re.compile(
'^ftp://([A-Za-z0-9_-]+:[A-Za-z0-9_-]+@)?(.*)$')
match_valid_ftp = valid_ftp_url.match(url)
invalid_ftp_msg = "Invalid ftp url: %s. Expected: ftp://[user:password@]host/path_to_file" % url
if match_valid_ftp is None:
raise Exception(invalid_ftp_msg)
url_without_scheme = sel_url[2:]
if sel_url is None:
raise Exception(invalid_ftp_msg)
sel_host, sel_path = splithost(sel_url)
if sel_host is None:
raise Exception(invalid_ftp_msg)
sel_user, host = splituser(sel_host)
if not sel_user is None:
sel_user, sel_password = splitpasswd(sel_user)
if not sel_user is None and len(
sel_user) > 0 and not sel_password is None and len(
sel_password) > 0:
# reconstruct url
url = "ftp://%s:%s@%s%s" % (sel_user, sel_password, host,
sel_path)
if user == None:
user = User(sel_user, sel_password, url)
logger.log(
logging.DEBUG, 'Adding user: '******' into content password manager')
else:
user.setUsername(sel_user)
user.setPassword(sel_password)
logger.log(
logging.DEBUG, 'Updating user credential for: ' +
user.getUsername() + ' in content password manager')
else:
if not user is None:
url = "ftp://%s:%s@%s%s" % (user.getUsername(),
user.getPassword(), host, sel_path)
if user != None:
user.setRetryCount(user.getRetryCount() + 1)
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, user.getUsername(), user.getPassword())
authhandler = urllib2.HTTPBasicAuthHandler(passman)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
authhandler)
else:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
r = None
localFile = None
try:
if verbose:
print '\tfetching ' + url + '...'
# timeout after 60 seconds of blocking operations like the connection attempt
r = opener.open(url, timeout=timeout)
if hasattr(options, 'checkuri') and options.checkuri:
if r.code == 200:
sys.exit(0)
if not destFile:
filename = url.split('/')[-1]
else:
filename = destFile
if filename.strip() == '':
sys.stderr.write(
'Error: URI does not contain the filename for downloading.\nExiting.\n'
)
logger.log(logging.DEBUG, 'program exists with error code 1\n')
sys.exit(1)
if dirPrefix != None:
filenamePath = os.path.join(workingdir, dirPrefix, filename)
else:
filenamePath = os.path.join(workingdir, filename)
filenamePath = os.path.abspath(filenamePath)
logger.log(logging.DEBUG, 'downloading file to ' + filenamePath)
dirname = os.path.dirname(filenamePath)
if not os.path.exists(dirname):
os.makedirs(dirname)
localFile = open(filenamePath, 'w')
localFile.write(r.read())
cj.save(cookiesFilePath)
return filenamePath
except urllib2.HTTPError, e:
if hasattr(options, 'checkuri') and options.checkuri:
sys.exit(1)
if hasattr(e, 'code') and e.code == 401:
if user != None and user.getUri() == url:
sys.stderr.write(
'Error: Invalid username and password combination for: ' +
url + '\n')
if user.getRetryCount() > 3:
sys.stderr.write(
'Error: Max authentication retry count reached.\nExiting.\n'
)
logger.log(logging.DEBUG,
'program exists with error code 2\n')
sys.exit(2)
if (hasattr(options, 'noninteractive') and options.noninteractive):
sys.stderr.write('Error: Authentication failed. Exiting.\n')
logger.log(logging.DEBUG, 'program exists with error code 2\n')
sys.exit(2)
else:
username = raw_input('Please enter username: '******'Please enter password: '******'Adding user: '******' into content password manager')
passwordManager.addUser(user)
else:
user.setUsername(username)
user.setPassword(password)
logger.log(
logging.DEBUG, 'Updating user: '******' in content password manager')
passwordManager.saveToken()
return downloadFileUsingURLLib(url, workingdir, dirPrefix,
timeout, destFile)
else:
if e.code != 404 or (hasattr(options, 'missingok')
and not options.missingok and e.code == 404):
sys.stderr.write('Receive HTTP Error for: ' + url + '\n')
if hasattr(e, 'code'):
sys.stderr.write('error code: ' + str(e.code) + '\n')
if hasattr(e, 'msg'):
sys.stderr.write('error message: ' + str(e.msg) + '\n')
if hasattr(e, 'reason'):
sys.stderr.write('error reason: ' + (e.reason) + '\n')
sys.stderr.write('Exiting.\n')
logger.log(logging.DEBUG, 'program exists with error code 3\n')
sys.exit(3)
elif e.code == 404 and (hasattr(options, 'missingok')
and options.missingok) and verbose:
logger.log(logging.WARNING,
'Failed to download file. File not found ' + url)
def smb_open(self, req):
global USE_NTLM, MACHINE_NAME
host = req.get_host()
if not host:
raise urllib2.URLError('SMB error: no host given')
host, port = splitport(host)
if port is None:
port = 139
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
passwd = passwd or ''
myname = MACHINE_NAME or self.generateClientMachineName()
n = NetBIOS()
names = n.queryIPForName(host)
if names:
server_name = names[0]
else:
raise urllib2.URLError(
'SMB error: Hostname does not reply back with its machine name'
)
path, attrs = splitattr(req.get_selector())
if path.startswith('/'):
path = path[1:]
dirs = path.split('/')
dirs = map(unquote, dirs)
service, path = dirs[0], '/'.join(dirs[1:])
try:
conn = SMBConnection(user,
passwd,
myname,
server_name,
use_ntlm_v2=USE_NTLM)
conn.connect(host, port)
if req.has_data():
data_fp = req.get_data()
filelen = conn.storeFile(service, path, data_fp)
headers = "Content-length: 0\n"
fp = StringIO("")
else:
fp = self.createTempFile()
file_attrs, retrlen = conn.retrieveFile(service, path, fp)
fp.seek(0)
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except Exception, ex:
raise urllib2.URLError, ('smb error: %s' % ex), sys.exc_info()[2]
def _parse_proxy(proxy):
"""Return (scheme, user, password, host/port) given a URL or an authority.
If a URL is supplied, it must have an authority (host:port) component.
According to RFC 3986, having an authority component means the URL must
have two slashes after the scheme:
>>> _parse_proxy('file:/ftp.example.com/')
Traceback (most recent call last):
ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
The first three items of the returned tuple may be None.
Examples of authority parsing:
>>> _parse_proxy('proxy.example.com')
(None, None, None, 'proxy.example.com')
>>> _parse_proxy('proxy.example.com:3128')
(None, None, None, 'proxy.example.com:3128')
The authority component may optionally include userinfo (assumed to be
username:password):
>>> _parse_proxy('joe:[email protected]')
(None, 'joe', 'password', 'proxy.example.com')
>>> _parse_proxy('joe:[email protected]:3128')
(None, 'joe', 'password', 'proxy.example.com:3128')
Same examples, but with URLs instead:
>>> _parse_proxy('http://proxy.example.com/')
('http', None, None, 'proxy.example.com')
>>> _parse_proxy('http://proxy.example.com:3128/')
('http', None, None, 'proxy.example.com:3128')
>>> _parse_proxy('http://*****:*****@proxy.example.com/')
('http', 'joe', 'password', 'proxy.example.com')
>>> _parse_proxy('http://*****:*****@proxy.example.com:3128')
('http', 'joe', 'password', 'proxy.example.com:3128')
Everything after the authority is ignored:
>>> _parse_proxy('ftp://*****:*****@proxy.example.com/rubbish:3128')
('ftp', 'joe', 'password', 'proxy.example.com')
Test for no trailing '/' case:
>>> _parse_proxy('http://*****:*****@proxy.example.com')
('http', 'joe', 'password', 'proxy.example.com')
"""
scheme, r_scheme = splittype(proxy)
if not r_scheme.startswith("/"):
# authority
scheme = None
authority = proxy
else:
# URL
if not r_scheme.startswith("//"):
raise ValueError("proxy URL with no authority: %r" % proxy)
# We have an authority, so for RFC 3986-compliant URLs (by ss 3.
# and 3.3.), path is empty or starts with '/'
end = r_scheme.find("/", 2)
if end == -1:
end = None
authority = r_scheme[2:end]
userinfo, hostport = splituser(authority)
if userinfo is not None:
user, password = splitpasswd(userinfo)
else:
user = password = None
return scheme, user, password, hostport
