def processRequest(self,method=None,url=None,data="",headers={}):
conf = desktop.Config()
if not conf['proxy']:
self.proxy_host = None
self.proxy_port = None
else:
self.proxy_host = conf['proxy']['proxy']
self.proxy_port = conf['proxy']['proxy_port']
socket.setdefaulttimeout(self.http_timeout)
(protocol,resource) = urllib.splittype(url)
(hostport,path) = urllib.splithost(resource)
connexion = None
if protocol.lower() == "http":
(host,port) = urllib.splitnport(hostport, 80)
import httplib
if self.proxy_host != None and self.proxy_port != None :
connexion = HTTPConnection(self.proxy_host, self.proxy_port, timeout=self.http_timeout)
path = url
else:
connexion = HTTPConnection(host, port, timeout=self.http_timeout)
elif protocol.lower() == "https" :
(host,port) = urllib.splitnport(hostport, 443)
connexion = HTTPSConnection(host, port)
if self.proxy_host != None and self.proxy_port != None :
connexion.http_proxy = [self.proxy_host,
self.proxy_port]
else:
assert False, "Unhandled Protocol, please use HTTP or HTTPS"
connexion.connect()
connexion.request(method, path, body=data, headers=headers)
response = connexion.getresponse()
return response
def main():
optlist, args = getopt.getopt(sys.argv[1:], 'sd:')
shared = 0
depth = 'infinity'
for opt, value in optlist:
if opt == '-d':
if value not in ['0', '1', 'infinity']:
print 'ERROR: "%s" is not a valid depth' % value
sys.exit(1)
depth = value
elif opt == '-s':
shared = 1
if len(args) < 2 or len(args) > 3:
usage()
command = args[0]
if not commands.has_key(command):
usage()
url = args[1]
extra = args[2:]
scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
host, port = urllib.splitnport(netloc, 80)
conn = davlib.DAV(host, port)
commands[command](conn, path, shared, depth, extra)
def request(self, method, url, body=None, headers=None):
"""
Send an HTTP or HTTPS request, as appropriate; and parse the request
URL in case it's a proxied request.
"""
protocol = self.scheme
headers = headers or {}
if self.is_proxy:
# if it's a proxied connection, a full URL is passed in, so parse it
protocol, remainder = urllib.splittype(url)
if protocol not in (HTTP_SCHEME, HTTPS_SCHEME):
raise ValueError('Unsupported URL protocol: %s' % url)
host, remainder = urllib.splithost(remainder)
host, port = urllib.splitnport(host)
port = port or self._ports[protocol]
self._real_host = host
self._real_port = port
if protocol == HTTP_SCHEME:
httplib.HTTPConnection.request(self, method, url, body, headers)
else: # HTTPS_SCHEME
httplib.HTTPSConnection.request(self, method, url, body, headers)
def run(self, netloc='localhost:8000', reload=True, log=True):
"""Run the CherryPy server."""
from django.conf import settings
from django.core.handlers.wsgi import WSGIHandler
from paste.translogger import TransLogger
host, port = urllib.splitnport(netloc, 80)
host = socket.gethostbyname(host)
cherrypy.config.update({
'server.socket_host': host,
'server.socket_port': port,
'log.screen': log,
'engine.autoreload_on': reload,
})
self.cfg_assets(settings.MEDIA_URL, settings.MEDIA_ROOT)
self.cfg_assets(settings.STATIC_URL, settings.STATIC_ROOT)
self.cfg_favicon(settings.STATIC_ROOT)
app = WSGIHandler()
app = TransLogger(app, logger_name='cherrypy.access',
setup_console_handler=False)
if self.domains:
app = cherrypy.wsgi.VirtualHost(app, self.domains)
cherrypy.tree.graft(app)
cherrypy.engine.start()
cherrypy.engine.block()
def _parse_url(self, url):
""" Returns scheme, host, port, path. """
scheme, netloc, path, params, query, frag = rhnLib.parseUrl(url)
host, port = urllib.splitnport(netloc)
if (port <= 0):
port = None
return scheme, host, port, path
def getChild (self, path, request, ) :
_host_orig = urllib.splitport(request.getHeader("host"), )[0]
_key = "%s:%s" % ("https" if request.isSecure() else "http", _host_orig, )
try :
_to = self._config.get(_key, "to", )
except ConfigParser.NoSectionError :
return BadRequestErrorPage()
_p = urlparse.urlsplit(_to, )
(_host, _port, ) = urllib.splitnport(_p.netloc, 443 if _p.scheme == "https" else 80, )
if not self._without_x_forwarded_for :
_headers = request.getAllHeaders()
_xf = ("%s, " % self.RE_REMOVE_COMMA.sub(
"", _headers.get('x-forwarded-for')).strip()
) if "x-forwarded-for" in _headers else ""
_x_forwarded_for = _xf + request.client.host
_x_forwarded_proto = "https" if request.isSecure() else "http"
request.received_headers['x-forwarded-for'] = _x_forwarded_for
request.received_headers['x-forwarded-proto'] = _x_forwarded_proto
request.received_headers['host-original'] = _host_orig
request.content.seek(0, 0)
return ReverseProxyResource(
_host,
_port,
"/" + path if path else "/",
reactor=FakeReactor(self._timeout, ),
)
def __init__(self, url, _client=None):
Transport.__init__(self, url)
(scheme, _, loc, _, _) = urlparse.urlsplit(url)
assert scheme == "git"
hostport, self._path = urllib.splithost(loc)
(self._host, self._port) = urllib.splitnport(hostport, git.protocol.TCP_GIT_PORT)
self._client = _client
def _parse_url(url):
""" Returns scheme, host, port, path. """
scheme, netloc, path, _params, _query, _frag = rhnLib.parseUrl(url)
host, port = urllib.splitnport(netloc)
if port <= 0:
port = None
return scheme, host, port, path
def __init__(self, hub, url):
self.hub = hub
parsed = urlparse.urlsplit(url)
assert parsed.query == ''
assert parsed.fragment == ''
default_port = 443 if parsed.scheme == 'https' else 80
host, port = urllib.splitnport(parsed.netloc, default_port)
self.socket = self.hub.tcp.connect(host=host, port=port)
# TODO: this shouldn't block on the SSL handshake
if parsed.scheme == 'https':
# TODO: what a mess
conn = self.socket.sender.fd.conn
conn = ssl.wrap_socket(conn)
conn.setblocking(0)
self.socket.sender.fd.conn = conn
self.socket.recver.sep = '\r\n'
self.agent = 'vanilla/%s' % vanilla.meta.__version__
self.default_headers = dict([
('Accept', '*/*'),
('User-Agent', self.agent),
('Host', parsed.netloc), ])
self.requests = self.hub.router().pipe(self.hub.queue(10))
self.requests.pipe(self.hub.consumer(self.writer))
self.responses = self.hub.router().pipe(self.hub.queue(10))
self.responses.pipe(self.hub.consumer(self.reader))
def _connect_ssh(self, host):
(user, host) = urllib.splituser(host)
if user is not None:
(user, password) = urllib.splitpassword(user)
else:
password = None
(host, port) = urllib.splitnport(host, 22)
self._tunnel = get_ssh_vendor().connect_ssh(user, password, host, port, ["svnserve", "-t"])
return (self._tunnel.recv, self._tunnel.send)
def makeService(self, options):
"""
Constructs a lobber storage node service
"""
lobber = LobberClient(options['lobberUrl'],
options['lobberKey'],
options['torrentDir'].rstrip(os.sep),
options['announceUrl'])
transmission = TransmissionClient(options['transmissionRpc'],
options['transmissionDownloadsDir'])
torrentDownloader = TorrentDownloader(lobber,transmission,
options.destinations,
options['trackerProxyTrackerUrl'],
options['trackerProxyListenOn'])
stompService = internet.TCPClient(options['stomp_host'],
options['stomp_port'],
torrentDownloader)
self.getter = {}
for url in options.urls:
log.msg("Pulling RSS/Torrent from "+url)
self.getter[url] = task.LoopingCall(torrentDownloader.url_handler.load_url,url)
self.getter[url].start(30,True)
transmissionSweeper = TransmissionSweeper(lobber, transmission,
remove_limit=options['removeLimit'])
self.sweeper = task.LoopingCall(transmissionSweeper.clean_done)
reactor.callLater(30/2, self.sweeper.start, 30, True)
if options['dropbox']:
dropboxWatcher = DropboxWatcher(lobber,transmission,
options['dropbox'],
register=options['register'],
acl=options['acl'],
move=not options['keepData'])
self.dropbox = task.LoopingCall(dropboxWatcher.watch_dropbox)
self.dropbox.start(5,True)
if options['trackerProxyTrackerUrl']:
netloc, path = urlparse(options['trackerProxyTrackerUrl'])[1:3]
tracker_host, tracker_port = splitnport(netloc, 443)
proxy = server.Site(
ReverseProxyTLSResource(
tracker_host,
tracker_port,
'',
#path_rewrite=[['[^\?]+', path]],
tls=True, # FIXME: Base on urlparse()[0].
headers={'X_LOBBER_KEY': options['lobberKey']}))
bindto = options['trackerProxyListenOn'].split(':')
bindto_host = bindto[0]
bindto_port = int(bindto[1])
reactor.listenTCP(bindto_port, proxy, interface=bindto_host)
return stompService
def get_host(self):
_Request.get_host(self)
if self.port is None:
if self.get_type() == 'http':
default = 80
elif self.get_type() == 'https':
default = 443
else:
raise ValueError('Unsupported type.', self.get_type())
self.host, self.port = splitnport(self.host, default)
return self.host
def remotefile_from_str(s):
import urllib
protocol, rest = s.split("://")
host, rest = urllib.splithost("//" + rest)
host, port = urllib.splitnport(host, defport=21)
file, t = urllib.splitquery(rest)
s = Server(hostname=host, port=port)
dir, fn = os.path.split(file)
rf = File(filename=fn, dir=dir, type="f", server=s)
return rf
def fetch(self, command, **params):
"""
@return object
"""
# Perform HTTP request
request_body = {}
request_body["Action"] = command
request_body["Version"] = self.environment.api_version
request_body['AuthVersion'] = '3'
if self.environment.env_id and self.environment.env_id != 'None':
request_body['EnvID'] = self.environment.env_id
#request_body['SysDebug'] = '1'
if {} != params :
for key, value in params.items():
if isinstance(value, dict):
for k,v in value.items():
request_body['%s[%s]'%(key,k)] = v
else:
request_body[key] = value
if not self.environment.auth_type or self.environment.auth_type == 'password':
request_body["KeyID"] = self.environment.key_id
signature, timestamp = sign_http_request_v3(request_body, self.environment.key_id, self.environment.key)
request_body["TimeStamp"] = timestamp
request_body["Signature"] = signature
elif self.environment.auth_type == 'ldap':
request_body["Login"] = self.environment.ldap_login
request_body["AuthType"] = self.environment.auth_type
password =self.environment.ldap_password or os.environ.get('SCALR_API_LDAP_PASSWORD')
if not password:
password = getpass.getpass('Password: '******'POST URL: \n%s' % self.environment.url)
self._logger.debug('POST DATA: \n%s' % post_data)
response = None
try:
req = Request(self.environment.url, post_data, {})
response = urlopen(req)
except URLError, e:
if isinstance(e, HTTPError):
resp_body = e.read() if e.fp is not None else ""
raise ScalrAPIError("Request failed. %s. URL: %s. Service message: %s" % (e, self.environment.url, resp_body))
else:
host, port = splitnport(req.host, req.port or 443)
raise ScalrAPIError("Cannot connect to Scalr on %s:%s. %s"
% (host, port, str(e)))
def __init__(self, host, transport, close_callable=None):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.host = host
self.getparser = transport.getparser
self.close_callable = close_callable
self.deferred = None
self.waiting_on_response = False
self.rx_buf = ""
self.tx_buf = ""
# Connect to host and POST request
self.connect(urllib.splitnport(host, 80))
def __init__(self, token, uri_prefix=DEFAULT_URI_PREFIX, log=Log()):
"""Sets up access to a service that provides the Google Meter API.
Args:
token: AuthSub token to use for all requests
uri_prefix: URI prefix under which feeds are located
log: Log object to which messages will be logged
"""
self.token = token
self.scheme, hostport, self.path, _, _, _ = urlparse.urlparse(uri_prefix)
default_port = {'http': 80, 'https': 443}[self.scheme]
self.host, self.port = urllib.splitnport(hostport, default_port)
self.log = log
def savefilename(self, url):
type, rest = urllib.splittype(url)
host, path = urllib.splithost(rest)
while path[:1] == "/": path = path[1:]
user, host = urllib.splituser(host)
host, port = urllib.splitnport(host)
host = string.lower(host)
if not path or path[-1] == "/":
path = path + "index.html"
if os.sep != "/":
path = string.join(string.split(path, "/"), os.sep)
path = os.path.join(host, path)
return path
def savefilename(self, url):
type, rest = urllib.splittype(url)
host, path = urllib.splithost(rest)
path = path.lstrip("/")
user, host = urllib.splituser(host)
host, port = urllib.splitnport(host)
host = host.lower()
if not path or path[-1] == "/":
path = path + "index.html"
if os.sep != "/":
path = os.sep.join(path.split("/"))
path = os.path.join(host, path)
return path
def savefilename(self, url):
type, rest = urllib.splittype(url)
host, path = urllib.splithost(rest)
path = path.lstrip("/")
user, host = urllib.splituser(host)
host, port = urllib.splitnport(host)
host = host.lower()
if not path or path[-1] == "/":
path = path + "index.html"
if os.sep != "/":
path = os.sep.join(path.split("/"))
if os.name == "mac":
path = os.sep + path
path = os.path.join(host, path)
return path
def _connect(self, host):
(host, port) = urllib.splitnport(host, SVN_PORT)
sockaddrs = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM, 0, 0)
self._socket = None
for (family, socktype, proto, canonname, sockaddr) in sockaddrs:
try:
self._socket = socket.socket(family, socktype, proto)
self._socket.connect(sockaddr)
except socket.error, err:
if self._socket is not None:
self._socket.close()
self._socket = None
continue
break
def savefilename(self, url):
type, rest = urllib.splittype(url)
host, path = urllib.splithost(rest)
while path[:1] == "/":
path = path[1:]
user, host = urllib.splituser(host)
host, port = urllib.splitnport(host)
host = string.lower(host)
if not path or path[-1] == "/":
path = path + "index.html"
if os.sep != "/":
path = string.join(string.split(path, "/"), os.sep)
if os.name == "mac":
path = os.sep + path
path = os.path.join(host, path)
return path
def openHotlineURL(url, nickname=None):
urlparse.uses_netloc.append('hotline')
schema, host, path, query, fragment = urlparse.urlsplit(url)
assert schema == 'hotline'
assert query == ''
assert fragment == ''
kws = {}
if nickname:
kws['nickname'] = nickname
if host.count('@'):
userstr, host = urllib.splituser(host)
kws['username'], kws['password'] = urllib.splittype(userstr)
kws['cwd'] = path
cc = ClientCreator(reactor, HotlineClient, **kws)
dd = cc.connectTCP(*urllib.splitnport(host, 5500))
return dd
def parseHotlineURL(url):
"""
@returns: ( (hostname, port,), kws, )
"""
urlparse.uses_netloc.append('hotline')
schema, host, path, query, fragment = urlparse.urlsplit(url)
assert schema == 'hotline'
assert query == ''
assert fragment == ''
kws = {}
if host.count('@'):
userstr, host = urllib.splituser(host)
kws['username'], kws['password'] = urllib.splittype(userstr)
kws['cwd'] = path
return ( urllib.splitnport(host, 5500), kws,)
def ParseUrl(value):
type = ""
host = ""
port = ""
path = ""
username = ""
password = ""
type, value = splittype(value or "")
value, path = splithost(value or "")
usps, value = splituser(value or "")
if usps:
username, password = splitpasswd(usps)
host, port = splitnport(value or "", None)
return type or "", host or "", str(
port) if port else "", path or "", username or "", password or ""
def _parse_host (h, ) :
if not h.startswith('serf://') :
h = 'serf://%s' % h
_parsed = list(urlparse.urlparse(h, ), )
if _parsed[0] not in ('serf', ) :
raise _exceptions.InvalidHostURL('invalid host url, `%s`.' % h, )
# the `urlparse` module in `travis-cis` does not understand the
# non-standard scheme like `serf'.
_parsed[0] = 'http'
if not _parsed[2].startswith('/') :
_parsed[2] = '/' + _parsed[2]
_parsed = urlparse.urlparse(urlparse.urlunparse(_parsed, ), )
_host = map(
lambda x : None if x in ('', -1, ) else x,
urllib.splitnport(_parsed.netloc, defport=7373, ),
)
if not _host[0] and not _host[1] :
raise _exceptions.InvalidHostURL('invalid host url, `%s`.' % h, )
if not _host[0] :
_host[0] = constant.DEFAULT_HOST
if not _host[1] :
_host[1] = constant.DEFAULT_PORT
_queries = dict()
if _parsed.query :
_queries = dict(map(
urllib.splitvalue,
filter(string.strip, _parsed.query.split('&'), ),
), )
if set(_queries.keys()) != AVAILABLE_HOST_URI_QUERY_KEYS :
raise _exceptions.InvalidHostURL(
'unknown key found, %s' % list(
set(_queries.keys() - AVAILABLE_HOST_URI_QUERY_KEYS), ),
)
_host.append(_queries, )
return _host
def urlSplit(url, defaultPort=None):
"""A function to split a URL in the format
<scheme>://<user>:<pass>@<host>:<port>/<path>;<params>#<fragment>
into a tuple
(<scheme>, <user>, <pass>, <host>, <port>, <path>, <params>, <fragment>)
Any missing pieces (user/pass) will be set to None.
If the port is missing, it will be set to defaultPort; otherwise, the port
should be a numeric value.
"""
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
userpass, hostport = urllib.splituser(netloc)
host, port = urllib.splitnport(hostport, None)
if userpass:
user, passwd = urllib.splitpasswd(userpass)
else:
user, passwd = None, None
return scheme, user, passwd, host, port, path, \
query or None, fragment or None
def urlSplit(url, defaultPort = None):
"""A function to split a URL in the format
<scheme>://<user>:<pass>@<host>:<port>/<path>;<params>#<fragment>
into a tuple
(<scheme>, <user>, <pass>, <host>, <port>, <path>, <params>, <fragment>)
Any missing pieces (user/pass) will be set to None.
If the port is missing, it will be set to defaultPort; otherwise, the port
should be a numeric value.
"""
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
userpass, hostport = urllib.splituser(netloc)
host, port = urllib.splitnport(hostport, None)
if userpass:
user, passwd = urllib.splitpasswd(userpass)
else:
user, passwd = None, None
return scheme, user, passwd, host, port, path, \
query or None, fragment or None
def check_args(options, args):
if not args:
return
# someway to test if it is a SIP URI
if 'sip:' in args[0]:
setattr(options, 'target_uri', args[0])
userpass, hostport = urllib.splituser(args[0])
if hostport[:4] == 'sip:': hostport = hostport[4:]
host, port = urllib.splitnport(hostport, 5060)
# XXX Overwrites!
setattr(options, 'target_addr', (host, port))
if userpass:
user, passwd = urllib.splitpasswd(userpass)
user and setattr(options, 'user', user)
passwd and setattr(options, 'passwd', passwd)
def _getconnection(netloc):
"""Return a FTP connection object to the specified server."""
# NOTE: this method is not thread safe
if _ftpconnections.has_key(netloc):
return _ftpconnections[netloc]
# split url into useful parts
(userpass, host) = urllib.splituser(netloc)
if userpass is not None:
(user, passwd) = urllib.splitpasswd(userpass)
else:
(user, passwd) = ('anonymous', '')
(host, port) = urllib.splitnport(host, ftplib.FTP_PORT)
# initialize a new connection
ftp = ftplib.FTP()
debugio.debug('schemes.ftp._getconnection(): CONNECT: '+ftp.connect(host, port))
debugio.debug('schemes.ftp._getconnection(): LOGIN: '+ftp.login(user, passwd))
_ftpconnections[netloc] = ftp
return ftp
def check_args(options, args):
if not args:
return
# someway to test if it is a SIP URI
if 'sip:' in args[0]:
setattr(options, 'target_uri', args[0])
userpass, hostport = urllib.splituser(args[0])
if hostport[:4] == 'sip:': hostport = hostport[4:]
host, port = urllib.splitnport(hostport, 5060)
# XXX Overwrites!
setattr(options, 'target_addr', (host, port))
if userpass:
user, passwd = urllib.splitpasswd(userpass)
user and setattr(options, 'user', user)
passwd and setattr(options, 'passwd', passwd)
def _getProxyURL(self, subpath='', query=''):
# Helper method to construct an URL appropriate for proxying a request.
# This makes sure that URLs generated by absolute_url at a remote site
# will be always towards the proxy method again.
#
# Note that the code assumes that VirtualHostBase is visible. The setting
# of a front-end server must allow this.
#
# This should generate an URL like this:
#
# http://remotehost:9080/VirtualHostBase/http/localhost:8080/VirtualHostRoot/_vh_erp5/_vh_portal_wizard/_vh_proxy/erp5/person_module/2
part_list = []
server_url = self.getServerUrl().rstrip('/')
part_list.append(server_url)
part_list.append('VirtualHostBase')
portal_url = self.getPortalObject().absolute_url()
scheme, rest = urllib.splittype(portal_url)
addr, path = urllib.splithost(rest)
host, port = urllib.splitnport(addr, scheme == 'http' and 80 or 443)
part_list.append(scheme)
part_list.append('%s:%s' % (host, port))
part_list.append('VirtualHostRoot')
method_path = self.absolute_url_path() + '/proxy'
part_list.extend(('_vh_' + p for p in method_path.split('/') if p))
server_root = self.getServerRoot().strip('/')
if isinstance(subpath, (list, tuple)):
subpath = '/'.join(subpath)
if not subpath.startswith(server_root):
part_list.append(server_root)
part_list.append(subpath)
url = '/'.join((p for p in part_list if p))
if query:
url = url + '?' + query
return url
def _getProxyURL(self, subpath='', query=''):
# Helper method to construct an URL appropriate for proxying a request.
# This makes sure that URLs generated by absolute_url at a remote site
# will be always towards the proxy method again.
#
# Note that the code assumes that VirtualHostBase is visible. The setting
# of a front-end server must allow this.
#
# This should generate an URL like this:
#
# http://remotehost:9080/VirtualHostBase/http/localhost:8080/VirtualHostRoot/_vh_erp5/_vh_portal_wizard/_vh_proxy/erp5/person_module/2
part_list = []
server_url = self.getServerUrl().rstrip('/')
part_list.append(server_url)
part_list.append('VirtualHostBase')
portal_url = self.getPortalObject().absolute_url()
scheme, rest = urllib.splittype(portal_url)
addr, path = urllib.splithost(rest)
host, port = urllib.splitnport(addr, scheme == 'http' and 80 or 443)
part_list.append(scheme)
part_list.append('%s:%s' % (host, port))
part_list.append('VirtualHostRoot')
method_path = self.absolute_url_path() + '/proxy'
part_list.extend(('_vh_' + p for p in method_path.split('/') if p))
server_root = self.getServerRoot().strip('/')
if isinstance(subpath, (list, tuple)):
subpath = '/'.join(subpath)
if not subpath.startswith(server_root):
part_list.append(server_root)
part_list.append(subpath)
url = '/'.join((p for p in part_list if p))
if query:
url = url + '?' + query
return url
def parse_remote_repository (uri, ) :
_pa = urlparse.urlsplit(uri)
(_a_user, _a_host, ) = urllib.splituser(_pa.netloc, )
_user, _password = None, None
if _a_user :
(_user, _password, ) = urllib.splitpasswd(_a_user, )
_defaut_port = _pa.scheme.lower() in ("svn+ssh", "ssh", ) and 22 or None
(_host, _port, ) = urllib.splitnport(_a_host, defport=_defaut_port, )
return dict(
host=_host,
port=_port,
scheme=_pa.scheme.lower(),
user=_user,
password=_password,
path=os.path.normpath(
_pa.path.strip().startswith("/") and _pa.path or ("/" + _pa.path)
),
)
def do_GET(self):
url = self.path
protocol,rest = urllib.splittype(url)
host,path = urllib.splithost(rest)
host,port = urllib.splitnport(host)
port = 80 if port < 0 else port
host_ip = socket.gethostbyname(host)
del self.headers['Proxy-Connection']
self.headers['Connection'] = 'close'
send_data = 'GET ' + path + ' HTTP/1.1' + '\r\n'
head = ''
for key, val in self.headers.items():
head = head + "%s: %s\r\n" % (key, val)
send_data = send_data + head + '\r\n'
client = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
block = ['.png','.css','.jpg','.gif','.js']
bb = True
for b in block:
if b in path:
bb = False
break
client.connect((host_ip,port))
client.send(send_data)
data = ''
while True:
tmp = client.recv(4096)
if not tmp:
break
data = data + tmp
if 'top.location.href = "/dir_login.asp"' in data:
data = data.replace('top.location.href = "/dir_login.asp"','')
if bb and path not in paths:
# loadinfo(send_data)
# loadinfo0(path+"\r\n")
paths.append(path)
print(send_data)
print(data+"\r\n\r\n")
client.close()
self.wfile.write(data)
def parse_uri(uri, **kwargs):
scheme, rest = urllib.splittype(uri)
host, rest = urllib.splithost(rest)
user, host = urllib.splituser(host)
if user:
username, password = urllib.splitpasswd(user)
else:
username = password = None
host, port = urllib.splitnport(host)
path, query = urllib.splitquery(rest)
if query:
kwargs.update(dict(cgi.parse_qsl(query)))
return dict(
scheme=scheme,
host=host,
username=username,
password=password,
port=port,
path=path,
query=kwargs)
def add_server(self, netloc, path, config):
"""Add a new CherryPy Application for a Virtual Host.
Creates a new CherryPy WSGI Server instance if the host resolves
to a different IP address or port.
"""
from cherrypy._cpwsgi_server import CPWSGIServer
from cherrypy.process.servers import ServerAdapter
host, port = urllib.splitnport(netloc, 80)
host = socket.gethostbyname(host)
bind_addr = (host, port)
if bind_addr not in self.servers:
self.servers.append(bind_addr)
server = CPWSGIServer()
server.bind_addr = bind_addr
adapter = ServerAdapter(cherrypy.engine, server, server.bind_addr)
adapter.subscribe()
self.domains[netloc] = cherrypy.Application(
root=None, config={path.rstrip('/') or '/': config})
def do_GET(self):
url = self.path
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
path = rest
host, port = urllib.splitnport(host)
if port < 0: port = 80
ip = socket.gethostbyname(host)
del self.headers['Proxy-Connection']
self.headers['Connection'] = 'close'
send_data = 'GET ' + path + ' ' + self.protocol_version + '\r\n'
head = ''
for key, val in self.headers.items():
head = head + "%s: %s\r\n" % (key, val)
send_data = send_data + head + '\r\n'
print send_data
so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
so.connect((ip, port))
so.sendall(send_data)
data = ''
while 1:
try:
buf = so.recv(8129)
data += buf
except:
buf = None
finally:
if not buf:
so.close()
break
self.wfile.write(data)
def add_server(self, netloc, path, config):
"""Add a new CherryPy Application for a Virtual Host.
Creates a new CherryPy WSGI Server instance if the host resolves
to a different IP address or port.
"""
from cherrypy._cpwsgi_server import CPWSGIServer
from cherrypy.process.servers import ServerAdapter
host, port = urllib.splitnport(netloc, 80)
host = socket.gethostbyname(host)
bind_addr = (host, port)
if bind_addr not in self.servers:
self.servers.append(bind_addr)
server = CPWSGIServer()
server.bind_addr = bind_addr
adapter = ServerAdapter(cherrypy.engine, server, server.bind_addr)
adapter.subscribe()
self.domains[netloc] = cherrypy.Application(root=None,
config={path.rstrip('/') or '/': config})
def do_GET(self):
uri = self.path
# print uri
proto, rest = urllib.splittype(uri)
host, rest = urllib.splithost(rest)
# print host
path = rest
host, port = urllib.splitnport(host)
if port < 0:
port = 80
print host
host_ip = socket.gethostbyname(host)
print port
del self.headers['Proxy-Connection']
self.headers['Connection'] = 'close'
send_data = 'GET ' + path + ' ' + self.protocol_version + '\r\n'
head = ''
for key, val in self.headers.items():
head = head + "%s: %s\r\n" % (key, val)
send_data = send_data + head + '\r\n'
# print send_data
so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
so.connect((host_ip, port))
so.sendall(send_data)
# 因为采用非长连接，所以会关闭连接， recv 会退出
data = ''
while True:
tmp = so.recv(4096)
if not tmp:
break
data = data + tmp
# socprint data
so.close()
self.wfile.write(data)
def __init__(self, host, request_body, transport):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.rxBuf = ""
self.contentLen = 0
self.state = self.GETHDR
self.host = host
self.parser = transport.getparser()
self.request = request_body # for debug
hdr = "POST /RPC2 HTTP/1.0\r\n"\
"User-Agent: AsyncXMLRPC/Python\r\n"\
"Content-Type: text/xml\r\n"\
"Content-length: %d\r\n\r\n" % (len(request_body))
self.txBuf = hdr + request_body
# Setup async response handler
self.deferred = Deferred.Deferred()
# Connect to host and POST request
self.connect(urllib.splitnport(host, 80))
def parse_connection_string(cs):
ret = {}
if cs.startswith('st://'):
ret['scheme'] = 'st'
ret['use-ssl'] = False
ret['port'] = siptracklib.default_port
rest = cs[5:]
elif cs.startswith('sts://'):
ret['scheme'] = 'sts'
ret['use-ssl'] = True
ret['port'] = siptracklib.default_ssl_port
rest = cs[6:]
else:
rest = cs
rest, port = urllib.splitnport(rest, None)
if port:
ret['port'] = port
username, hostname = urllib.splituser(rest)
if username:
ret['username'] = username
if hostname:
ret['server'] = hostname
return ret
def sane_config(options):
if not options.target_uri and not options.to_uri:
options.target_uri = options.to_uri = '"siping" <sip:siping@localhost>'
elif options.target_uri and not options.to_uri:
options.to_uri = options.target_uri
elif options.to_uri and not options.target_uri:
options.target_uri = options.to_uri
#if not options.target_uri: options.target_uri = options.to_uri
if not options.from_uri: options.from_uri = options.to_uri
if not options.contacts_uri: options.contacts_uri.append(options.from_uri)
if not options.cseq_method:
options.cseq_method = options.method
if not hasattr(options, 'target_addr'):
hp = options.to_uri.find('sip:')
hp = options.to_uri[hp + 4:]
if '@' in hp:
up, hp = urllib.splituser(hp)
if hp[-1] == '>': hp = hp[:-1]
h, p = urllib.splitnport(hp, 5060)
if type(p) != int: p = int(p)
setattr(options, 'target_addr', (h, p))
if up:
u, p = urllib.splitpasswd(up)
setattr(options, 'username', u)
setattr(options, 'password', p)
if not options.target_addr[1]: # no port
options.target_addr = (options.target_addr[0], 5060)
#! /usr/bin/env python
def get_crawl_domain(url):
if isinstance(url, str):
protocol, __ = urllib.splittype(url)
host = urllib.splitnport(urllib.splithost(__)[0])
return host[0]
def parse_host(addr, is_win32=False, tls=False):
proto = "http+unix"
port = None
path = ''
if not addr and is_win32:
addr = DEFAULT_NPIPE
if not addr or addr.strip() == 'unix://':
return DEFAULT_UNIX_SOCKET
addr = addr.strip()
if addr.startswith('http://'):
addr = addr.replace('http://', 'tcp://')
if addr.startswith('http+unix://'):
addr = addr.replace('http+unix://', 'unix://')
if addr == 'tcp://':
raise errors.DockerException(
"Invalid bind address format: {0}".format(addr))
elif addr.startswith('unix://'):
addr = addr[7:]
elif addr.startswith('tcp://'):
proto = 'http{0}'.format('s' if tls else '')
addr = addr[6:]
elif addr.startswith('https://'):
proto = "https"
addr = addr[8:]
elif addr.startswith('npipe://'):
proto = 'npipe'
addr = addr[8:]
elif addr.startswith('fd://'):
raise errors.DockerException("fd protocol is not implemented")
else:
if "://" in addr:
raise errors.DockerException(
"Invalid bind address protocol: {0}".format(addr))
proto = "https" if tls else "http"
if proto in ("http", "https"):
address_parts = addr.split('/', 1)
host = address_parts[0]
if len(address_parts) == 2:
path = '/' + address_parts[1]
host, port = splitnport(host)
if port is None:
raise errors.DockerException("Invalid port: {0}".format(addr))
if not host:
host = DEFAULT_HTTP_HOST
else:
host = addr
if proto in ("http", "https") and port == -1:
raise errors.DockerException(
"Bind address needs a port: {0}".format(addr))
if proto == "http+unix" or proto == 'npipe':
return "{0}://{1}".format(proto, host).rstrip('/')
return "{0}://{1}:{2}{3}".format(proto, host, port, path).rstrip('/')
def parse_host(addr, is_win32=False, tls=False):
path = ''
port = None
host = None
# Sensible defaults
if not addr and is_win32:
return DEFAULT_NPIPE
if not addr or addr.strip() == 'unix://':
return DEFAULT_UNIX_SOCKET
addr = addr.strip()
parsed_url = urlparse(addr)
proto = parsed_url.scheme
if not proto or any([x not in string.ascii_letters + '+' for x in proto]):
# https://bugs.python.org/issue754016
parsed_url = urlparse('//' + addr, 'tcp')
proto = 'tcp'
if proto == 'fd':
raise errors.DockerException('fd protocol is not implemented')
# These protos are valid aliases for our library but not for the
# official spec
if proto == 'http' or proto == 'https':
tls = proto == 'https'
proto = 'tcp'
elif proto == 'http+unix':
proto = 'unix'
if proto not in ('tcp', 'unix', 'npipe', 'ssh'):
raise errors.DockerException(
"Invalid bind address protocol: {}".format(addr))
if proto == 'tcp' and not parsed_url.netloc:
# "tcp://" is exceptionally disallowed by convention;
# omitting a hostname for other protocols is fine
raise errors.DockerException(
'Invalid bind address format: {}'.format(addr))
if any([
parsed_url.params, parsed_url.query, parsed_url.fragment,
parsed_url.password
]):
raise errors.DockerException(
'Invalid bind address format: {}'.format(addr))
if parsed_url.path and proto == 'ssh':
raise errors.DockerException(
'Invalid bind address format: no path allowed for this protocol:'
' {}'.format(addr))
else:
path = parsed_url.path
if proto == 'unix' and parsed_url.hostname is not None:
# For legacy reasons, we consider unix://path
# to be valid and equivalent to unix:///path
path = '/'.join((parsed_url.hostname, path))
if proto in ('tcp', 'ssh'):
# parsed_url.hostname strips brackets from IPv6 addresses,
# which can be problematic hence our use of splitnport() instead.
host, port = splitnport(parsed_url.netloc)
if port is None or port < 0:
if proto != 'ssh':
raise errors.DockerException(
'Invalid bind address format: port is required:'
' {}'.format(addr))
port = 22
if not host:
host = DEFAULT_HTTP_HOST
# Rewrite schemes to fit library internals (requests adapters)
if proto == 'tcp':
proto = 'http{}'.format('s' if tls else '')
elif proto == 'unix':
proto = 'http+unix'
if proto in ('http+unix', 'npipe'):
return "{}://{}".format(proto, path).rstrip('/')
return '{0}://{1}:{2}{3}'.format(proto, host, port, path).rstrip('/')
def addr_callback(option, opt_str, value, parser):
host, port = urllib.splitnport(value, 5060)
setattr(parser.values, option.dest, (host, port))
def via_headers(msg):
msg += sip.utils.make_via()
if self.config.via:
msg.via.host, msg.via.port = urllib.splitnport(
self.config.via, 5060)
def main():
parser = argparse.ArgumentParser(
description='Geotag pictures in the given folder.')
parser.add_argument('folder',
help='The source with pictures to geotag.',
nargs='?',
default=None)
parser.add_argument('-d',
dest="download_url",
help='Download location data from url.',
nargs='?',
default=None)
parser.add_argument('-o',
dest="overwrite",
action="store_true",
help='Overwrite location tag if exists.')
parser.add_argument(
'-r',
dest="time_range",
help='Location query time range in minutes. (default 15)',
type=int,
default=15)
options = parser.parse_args()
if options.folder is None and options.download_url is None:
parser.print_help()
exit()
tagger = picgeotager.PicGotagger(options.time_range, options.overwrite)
if options.download_url is not None:
if not options.download_url.startswith("http://"):
options.download_url = "http://" + options.download_url
port = urllib.splitnport(
urllib.splithost(urllib.splittype(options.download_url)[1])[0])[1]
if port is None or port < 0:
options.download_url += ":" + str(default_port)
print "Downloading location data from", options.download_url
added = tagger.update_location_data(options.download_url)
print "Added", added, "locations to database."
tagged_count, already_tagged_count, error_count = (0, 0, 0)
if options.folder is not None:
options.folder = unicode(options.folder, "UTF-8")
if os.path.isfile(options.folder):
result = tagger.geotag_picture(options.folder)
if result == picgeotager.GeotagStatus.tagged:
tagged_count += 1
elif result == picgeotager.GeotagStatus.already_tagged:
already_tagged_count += 1
elif result == picgeotager.GeotagStatus.failed:
error_count += 1
else:
tagged_count, already_tagged_count, error_count = tagger.geotag_pictures(
options.folder)
print tagged_count, "pictures tagged."
print already_tagged_count, "pictures were already tagged."
print error_count, "errors."
def _spliturl(url):
scheme, part = url.split(':', 1)
host, path = urllib.splithost(part)
host, port = urllib.splitnport(host, 80)
return scheme, host, port, path
def display (self):
# Collect information useful for the various checks.
self.answers = {}
answers = self.troubleshooter.answers
if not answers['cups_queue_listed']:
return False
name = answers['cups_queue']
parent = self.troubleshooter.get_window ()
# Find out if this is a printer or a class.
try:
cups.setServer ('')
c = TimedOperation (cups.Connection, parent=parent).run ()
printers = TimedOperation (c.getPrinters, parent=parent).run ()
if printers.has_key (name):
self.answers['is_cups_class'] = False
queue = printers[name]
self.answers['cups_printer_dict'] = queue
else:
self.answers['is_cups_class'] = True
classes = TimedOperation (c.getClasses, parent=parent).run ()
queue = classes[name]
self.answers['cups_class_dict'] = queue
attrs = TimedOperation (c.getPrinterAttributes, (name,),
parent=parent).run ()
self.answers['local_cups_queue_attributes'] = attrs
except:
pass
if self.answers.has_key ('cups_printer_dict'):
cups_printer_dict = self.answers['cups_printer_dict']
uri = cups_printer_dict['device-uri']
(scheme, rest) = urllib.splittype (uri)
self.answers['cups_device_uri_scheme'] = scheme
if scheme in ["ipp", "http", "https"]:
(hostport, rest) = urllib.splithost (rest)
(host, port) = urllib.splitnport (hostport, defport=631)
self.answers['remote_server_name'] = host
self.answers['remote_server_port'] = port
elif scheme == "smb":
u = smburi.SMBURI (uri)
(group, host, share, user, password) = u.separate ()
new_environ = os.environ.copy()
new_environ['LC_ALL'] = "C"
if group:
args = ["nmblookup", "-W", group, host]
else:
args = ["nmblookup", host]
try:
p = TimedSubprocess (parent=parent,
timeout=5000,
args=args,
env=new_environ,
close_fds=True,
stdin=file("/dev/null"),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
result = p.run ()
self.answers['nmblookup_output'] = result
for line in result[0]:
if line.startswith ("querying"):
continue
spc = line.find (' ')
if (spc != -1 and
not line[spc:].startswith (" failed ")):
# Remember the IP address.
self.answers['remote_server_name'] = line[:spc]
break
except OSError:
# Problem executing command.
pass
elif scheme == "hp":
new_environ = os.environ.copy()
new_environ['LC_ALL'] = "C"
new_environ['DISPLAY'] = ""
try:
p = TimedSubprocess (parent=parent,
timeout=3000,
args=["hp-info", "-d" + uri],
close_fds=True,
env=new_environ,
stdin=file("/dev/null"),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.answers['hplip_output'] = p.run ()
except OSError:
# Problem executing command.
pass
r = cups_printer_dict['printer-type'] & cups.CUPS_PRINTER_REMOTE
self.answers['cups_printer_remote'] = (r != 0)
return False
