def _GetOpener(self):
"""Returns an OpenerDirector that supports cookies and ignores redirects.
Returns:
A urllib2.OpenerDirector object.
"""
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
if self.save_cookies:
self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
if os.path.exists(self.cookie_file):
try:
self.cookie_jar.load()
self.authenticated = True
StatusUpdate("Loaded authentication cookies from %s" %
self.cookie_file)
except (cookielib.LoadError, IOError):
# Failed to load cookies - just ignore them.
pass
else:
# Create an empty cookie file with mode 600
fd = os.open(self.cookie_file, os.O_CREAT, 0600)
os.close(fd)
# Always chmod the cookie file
os.chmod(self.cookie_file, 0600)
else:
# Don't save cookies across runs of update.py.
self.cookie_jar = cookielib.CookieJar()
opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
return opener
def _twitter_req(self, url, method, parameters):
req = oauth2.Request.from_consumer_and_token(self.oauth_consumer,
token = self.oauth_token,
http_method = \
self.http_method,
http_url = url,
parameters = parameters)
req.sign_request(self.signature_method_hmac_sha1, self.oauth_consumer,
self.oauth_token)
headers = req.to_header()
if (self.http_method == "POST"):
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
opener = urllib2.OpenerDirector()
opener.add_handler(self.http_handler)
opener.add_handler(self.https_handler)
response = opener.open(url, encoded_post_data)
return response
def sendRequest(self, content, method='GET'):
req = urllib2.Request(
'%s://%s:%d%s' %
(self.options.ssl and 'https' or 'http', self.options.remote[0],
self.options.remote[1], self.path), None, self.headers)
# not using the build_opener as i want to handle 302s...
# opener = urllib2.build_opener()
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPHandler())
req.add_header(
'Host', '%s:%d' % (self.options.remote[0], self.options.remote[1]))
if method not in 'GET':
req.get_method = lambda: method
if content is not None:
resp = opener.open(req, content)
else:
resp = opener.open(req)
info = resp.info()
data = resp.read()
def sedform(data):
for xform in self.options.responseSed:
data = xform(data)
return data
data = sedform(data)
headers = idict({k: sedform(v) for k, v in info.items()})
self.logResponse(resp, headers, data)
self.sendResponse(resp, headers, data)
def twitterreq(url, method, parameters):
req = oauth.Request.from_consumer_and_token(oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
headers = req.to_header()
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
opener.add_handler(https_handler)
response = opener.open(url, encoded_post_data)
return response
def _FetchUrl(self, url, post_data=None, parameters=None):
if post_data:
http_method = "POST"
else:
http_method = "GET"
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPSHandler())
if not post_data:
opener.addheaders.append(('Accept-Encoding', 'gzip'))
if http_method == "POST":
parameters = post_data.copy()
req = oauth.Request.from_consumer_and_token(self._oauth_consumer,
token=self._oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(self._signature_method_hmac_sha1,
self._oauth_consumer, self._oauth_token)
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
try:
response = opener.open(url, encoded_post_data)
url_data = self._UnGzip(response)
except urllib2.HTTPError, e:
print e
def threadedTwitterRequest(self):
"""
Method for downloading Twitter stream data in a new Thread.
"""
if not os.path.isfile(self.TwitterKeysFile):
tkMessageBox.showerror(
"ERROR",
"Twitter API credentials not filled. Use the Set/Update Twitter Credentials button to do so.",
parent=self.parent)
return
# note starting time
start_time = time.time()
#search_term = TweetSentimentTermEntry.get()
# read Twitter credentials
with open("Twitter_API_Keys", "r") as twitter_keys_file:
twitter_keys = twitter_keys_file.read().split("|")
#print twitter_keys
api_key = twitter_keys[0]
api_secret = twitter_keys[1]
access_token_key = twitter_keys[2]
access_token_secret = twitter_keys[3]
try:
_debug = 0
# oauth token objects
oauth_token = oauth.Token(key=access_token_key,
secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
# method = GET
http_method = "GET"
# handlers for HTTP and HTTPS
http_handler = urllib.HTTPHandler(debuglevel=_debug)
https_handler = urllib.HTTPSHandler(debuglevel=_debug)
## if search_term == "":
## url = "https://stream.twitter.com/1/statuses/sample.json"
## else:
## url = "https://api.twitter.com/1.1/search/tweets.json?q="
## url += search_term.strip().split()[0]
## print search_term
# API URL
url = "https://stream.twitter.com/1/statuses/sample.json"
print url
# optional parameters
parameters = []
# make the request object
req = oauth.Request.from_consumer_and_token(
oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
# sign the request
req.sign_request(signature_method_hmac_sha1, oauth_consumer,
oauth_token)
# get request headers
headers = req.to_header()
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
# initialize stream opener
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
opener.add_handler(https_handler)
# remove old cached Twitter stream
if os.path.isfile(self.TwitterStreamFile):
os.remove(self.TwitterStreamFile)
# make the request
response = opener.open(url, encoded_post_data)
print "Updating Twitter Stream ..."
# write response to cache file
for line in response:
self.var.set(
"Updating... This process takes 1-2 minutes to complete. Please wait ..."
)
# update for some time
if abs(time.time() - start_time) >= 60:
self.pb.pack_forget()
self.var.set("Update complete.")
return
with open(self.TwitterStreamFile, "a") as twitter_stream_file:
twitter_stream_file.write(line.strip() + os.linesep)
except:
print "EXCEPTION"
def __init__(self, req):
Exception.__init__(self, "Redirect")
self.req = req
class MyRedirectHandler(urllib2.HTTPRedirectHandler):
"""Throw an exception on redirects instead of continuing. The redirect will be handled in the main thread
so it can work with connection pooling."""
def redirect_request(self, req, fp, code, msg, headers, newurl):
new_req = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
if new_req:
raise Redirect(new_req)
# Our handler differs from the Python default in that:
# - we don't support file:// URLs
# - we don't follow HTTP redirects
_my_urlopen = urllib2.OpenerDirector()
for klass in [urllib2.ProxyHandler, urllib2.UnknownHandler, urllib2.HTTPHandler,
urllib2.HTTPDefaultErrorHandler, MyRedirectHandler,
urllib2.FTPHandler, urllib2.HTTPErrorProcessor, MyHTTPSHandler]:
_my_urlopen.add_handler(klass())
def download_in_thread(url, target_file, if_modified_since, notify_done):
try:
#print "Child downloading", url
if url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:'):
req = urllib2.Request(url)
if url.startswith('http:') and if_modified_since:
req.add_header('If-Modified-Since', if_modified_since)
src = _my_urlopen.open(req)
else:
raise Exception(_('Unsupported URL protocol in: %s') % url)
def GetAdminOpener(host,
user=None,
pwd=None,
otp_entry=None,
cookiejar_path=None):
"""Returns an OpenerDirector for retrieving administrative URLs.
Uses stored admin cookies if available, or prompts for authentication
credentials and authenticates with server otherwise.
Based on reitveld codereview script.
"""
opener = urllib2.OpenerDirector()
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
# TODO(spencer): remove the HTTP handler when we move to AsyncHTTPSTestCase.
# This is only for testing currently.
opener.add_handler(urllib2.HTTPHandler())
if cookiejar_path is None:
cookiejar_path = expanduser('~/.viewfinder_admin_cookie')
cookie_jar = cookielib.MozillaCookieJar(cookiejar_path)
if os.path.exists(cookiejar_path):
try:
cookie_jar.load()
logging.info('loaded admin authentication cookies from %s' %
cookiejar_path)
except:
# Otherwise, bad cookies; clear them.
os.unlink(cookiejar_path)
if not os.path.exists(cookiejar_path):
# Create empty file with correct permissions.
fd = os.open(cookiejar_path, os.O_CREAT, 0600)
os.close(fd)
# Always chmod to be sure.
os.chmod(cookiejar_path, 0600)
opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar))
class TornadoXSRFProcessor(urllib2.BaseHandler):
"""Add tornado's xsrf headers to outgoing requests."""
handler_order = urllib2.HTTPCookieProcessor.handler_order + 1
def http_request(self, request):
cookie_header = request.get_header('Cookie')
if cookie_header is not None and '_xsrf=' in cookie_header:
# We have an xsrf cookie in the cookie jar. Copy it into the X-Xsrftoken header.
request.add_unredirected_header(
'X-Xsrftoken',
re.match('_xsrf=([^;]+)', cookie_header).group(1))
else:
# No xsrf cookie, so just make one up. (this is currently the expected case because cookielib
# considers our xsrf cookie to be a "session" cookie and doesn't save it)
request.add_unredirected_header('X-Xsrftoken', 'fake_xsrf')
if cookie_header:
request.add_unredirected_header(
'Cookie', '_xsrf="fake_xsrf"; ' + cookie_header)
else:
request.add_unredirected_header('Cookie',
'_xsrf="fake_xsrf"')
return request
https_request = http_request
opener.add_handler(TornadoXSRFProcessor())
# Look for admin cookie. If it doesn't exist (or is expired), prompt
# and reauthenticate.
if len(cookie_jar) == 0 or \
any([c.is_expired() for c in cookie_jar if c.domain == host]):
if user is None or pwd is None or otp_entry is None:
user, pwd, otp_entry = _PromptForAdminCookie(user, pwd, otp_entry)
from viewfinder.backend.www.admin import admin_api
admin_api.Authenticate(opener, host, user, pwd, otp_entry)
cookie_jar.save()
logging.info('saved admin authentication cookies to %s' %
cookiejar_path)
return opener
sends the user to that location with the provided HTTP code.
"""
dest = cls.format_output_url(dest or "/")
response.status_int = code
response.headers['Location'] = dest
class EmbedHandler(urllib2.BaseHandler, urllib2.HTTPHandler,
urllib2.HTTPErrorProcessor,
urllib2.HTTPDefaultErrorHandler):
def http_redirect(self, req, fp, code, msg, hdrs):
to = hdrs['Location']
h = urllib2.HTTPRedirectHandler()
r = h.redirect_request(req, fp, code, msg, hdrs, to)
return embedopen.open(r)
http_error_301 = http_redirect
http_error_302 = http_redirect
http_error_303 = http_redirect
http_error_307 = http_redirect
embedopen = urllib2.OpenerDirector()
embedopen.add_handler(EmbedHandler())
def proxyurl(url):
r = urllib2.Request(url, None, {})
content = embedopen.open(r).read()
return content
def __init__(self,
proxy=None,
noverify=True,
follow_redirects=False,
headers={},
timeout=5,
cadata=None):
self.ctx = ssl.create_default_context(cadata=cadata)
if noverify:
self.ctx.check_hostname = False
self.ctx.verify_mode = ssl.CERT_NONE
self.proxy = None
tproxy = type(proxy)
if tproxy in (str, unicode):
proxyscheme = urlparse.urlparse(proxy)
scheme = proxyscheme.scheme.upper()
if scheme == 'SOCKS':
scheme = 'SOCKS5'
self.proxy = scheme, proxyscheme.hostname+(
':'+str(proxyscheme.port) if proxyscheme.port else ''), \
proxyscheme.username or None, \
proxyscheme.password or None
elif proxy in (True, None):
self.proxy = find_default_proxy()
elif hasattr(proxy, 'as_tuple'):
self.proxy = proxy.as_tuple()
else:
self.proxy = proxy
self.noverify = noverify
self.timeout = timeout
if not self.proxy:
handlers = [
StreamingHTTPHandler,
StreamingHTTPSHandler(context=self.ctx),
TCPReaderHandler(context=self.ctx)
]
else:
scheme, host, user, password = self.proxy
scheme = socks.PROXY_TYPES[scheme]
port = socks.DEFAULT_PORTS[scheme]
if ':' in host:
host, maybe_port = host.split(':')
try:
port = int(maybe_port)
except ValueError:
pass
sockshandler = SocksiPyHandler(
scheme,
host,
port,
user or None,
password or None,
context=self.ctx if self.noverify else None)
handlers = []
if scheme == socks.HTTP:
http_proxy = '{}:{}'.format(host, port)
if user and password:
http_proxy = '{}:{}@{}'.format(user, password, http_proxy)
http_proxy = 'http://' + http_proxy
handlers.append(urllib2.ProxyHandler({'http': http_proxy}))
handlers.append(StreamingHTTPHandler)
handlers.append(sockshandler)
if not follow_redirects:
handlers.append(NoRedirects)
handlers.append(UDPReaderHandler)
self.opener = urllib2.OpenerDirector()
for h in handlers:
if isinstance(h, (types.ClassType, type)):
h = h()
self.opener.add_handler(h)
if type(headers) == dict:
self.opener.addheaders = [(x, y) for x, y in headers.iteritems()]
else:
self.opener.addheaders = headers
def check_http_traffic(request):
settings.TOTAL_OF_REQUESTS = settings.TOTAL_OF_REQUESTS + 1
# Delay in seconds between each HTTP request
time.sleep(int(settings.DELAY))
if settings.SCHEME == 'https':
handle = httplib.HTTPSConnection
else:
handle = httplib.HTTPConnection
class do_connection(handle):
def request(self, method, url, body, headers):
info_msg = "The provided HTTP request headers: "
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_info_msg(info_msg)
if menu.options.traffic_file:
logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37)
header = method + " " + url
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
for item in headers.items():
header = item[0] + ": " + item[1]
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
if body:
header = body
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
if menu.options.traffic_file:
logs.log_traffic("\n\n")
if settings.SCHEME == 'https':
httplib.HTTPSConnection.request(self, method, url, body,
headers)
else:
httplib.HTTPConnection.request(self, method, url, body,
headers)
class connection_handler(urllib2.HTTPHandler, urllib2.HTTPSHandler):
if settings.SCHEME == 'https':
def https_open(self, req):
try:
return self.do_open(do_connection, req)
except Exception as err_msg:
try:
error_msg = str(err_msg.args[0]).split("] ")[1] + "."
except IndexError:
error_msg = str(err_msg.args[0]) + "."
if settings.INIT_TEST == True:
if settings.VERBOSITY_LEVEL < 2:
print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]"
else:
if settings.VERBOSITY_LEVEL < 1:
print ""
print settings.print_critical_msg(error_msg)
raise SystemExit()
else:
def http_open(self, req):
try:
return self.do_open(do_connection, req)
except Exception as err_msg:
try:
error_msg = str(err_msg.args[0]).split("] ")[1] + "."
except IndexError:
error_msg = str(err_msg.args[0]) + "."
if settings.INIT_TEST == True:
if settings.VERBOSITY_LEVEL < 2:
print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]"
else:
if settings.VERBOSITY_LEVEL < 1:
print ""
print settings.print_critical_msg(error_msg)
raise SystemExit()
if settings.REVERSE_TCP == False and settings.BIND_TCP == False:
opener = urllib2.OpenerDirector()
opener.add_handler(connection_handler())
response = False
current_attempt = 0
while not response and current_attempt <= settings.MAX_RETRIES:
try:
opener.open(request)
response = True
if settings.VERBOSITY_LEVEL < 2:
if current_attempt != 0:
info_msg = "Checking connection to the target URL... "
sys.stdout.write(settings.print_info_msg(info_msg))
sys.stdout.flush()
if settings.INIT_TEST == True:
print "[ " + Fore.GREEN + "SUCCEED" + Style.RESET_ALL + " ]"
if not settings.CHECK_INTERNET:
settings.INIT_TEST = False
except urllib2.URLError, err_msg:
if current_attempt == 0:
if settings.VERBOSITY_LEVEL < 2:
print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]"
try:
error_msg = str(err_msg.args[0]).split("] ")[1] + ". "
except IndexError:
error_msg = ""
error_msg += "Please wait while retring the request(s)."
print settings.print_critical_msg(error_msg)
warn_msg = "In case the provided target URL is valid, try to rerun with"
warn_msg += " the switch '--random-agent' and/or proxy switch."
print settings.print_warning_msg(warn_msg)
if settings.VERBOSITY_LEVEL >= 2 or current_attempt == 1:
info_msg = "Please wait while retring the request(s)."
print settings.print_info_msg(info_msg)
current_attempt = current_attempt + 1
time.sleep(3)
except httplib.BadStatusLine, err_msg:
if settings.VERBOSITY_LEVEL < 2:
print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]"
if len(err_msg.line) > 2:
print err_msg.line, err_msg.message
raise SystemExit()
except ValueError, err:
if settings.VERBOSITY_LEVEL < 2:
print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]"
err_msg = "Invalid target URL has been given."
print settings.print_critical_msg(err_msg)
raise SystemExit()
def __call__(self, value):
try:
super(RelativeURLValidator, self).__call__(value)
except ValidationError as e:
# Trivial case failed. Try for possible IDN domain
if value:
value = smart_text(value)
scheme, netloc, path, query, fragment = urlparse.urlsplit(
value)
try:
netloc = netloc.encode('idna') # IDN -> ACE
except UnicodeError: # invalid domain part
raise e
url = urlparse.urlunsplit(
(scheme, netloc, path, query, fragment))
super(RelativeURLValidator, self).__call__(url)
else:
raise
else:
url = value
if self.verify_exists:
broken_error = ValidationError(
_(u'This URL appears to be a broken link.'),
code='invalid_link')
if url.startswith('http://') or url.startswith('ftp://'):
headers = {
"Accept":
"text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
"Accept-Language": "en-us,en;q=0.5",
"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
"Connection": "close",
"User-Agent": self.user_agent,
}
url = url.encode('utf-8')
try:
req = urllib2.Request(url, None, headers)
req.get_method = lambda: 'HEAD'
#Create an opener that does not support local file access
opener = urllib2.OpenerDirector()
#Don't follow redirects, but don't treat them as errors either
error_nop = lambda *args, **kwargs: True
http_error_processor = urllib2.HTTPErrorProcessor()
http_error_processor.http_error_301 = error_nop
http_error_processor.http_error_302 = error_nop
http_error_processor.http_error_307 = error_nop
handlers = [
urllib2.UnknownHandler(),
urllib2.HTTPHandler(),
urllib2.HTTPDefaultErrorHandler(),
urllib2.FTPHandler(), http_error_processor
]
try:
import ssl
handlers.append(urllib2.HTTPSHandler())
except:
#Python isn't compiled with SSL support
pass
map(opener.add_handler, handlers)
if platform.python_version_tuple() >= (2, 6):
opener.open(req, timeout=10)
else:
opener.open(req)
except ValueError:
raise ValidationError(_(u'Enter a valid URL.'),
code='invalid')
except: # urllib2.URLError, httplib.InvalidURL, etc.
raise broken_error
else:
# Resolve the relative URL
try:
resolve(url)
except Http404:
raise broken_error
#Default opener, used for the password request
opener1 = urllib2.build_opener(
urllib2.HTTPBasicAuthHandler(passMan),
urllib2.HTTPCookieProcessor(),
urllib2.HTTPHandler(debuglevel=options.debug))
#No-redirect opener: Why am I building the opener in this
#strange way, instead of using the code above? Because in
#older versions of Python, redirects after a POST are not
#handled correctly. ELOG responds to a successful post with a
#redirect, which then hangs. Since we don't actually care
#about the redirection, I build an opener that doesn't include
#redirection (or a bunch of other default things). Should
#work fine-- ignores request to redirect and terminates.
opener2 = urllib2.OpenerDirector()
opener2.add_handler(urllib2.HTTPHandler(debuglevel=options.debug))
opener2.add_handler(urllib2.ProxyHandler())
opener2.add_handler(urllib2.HTTPBasicAuthHandler(passMan))
opener2.add_handler(urllib2.HTTPCookieProcessor())
opener2.add_handler(urllib2.HTTPDefaultErrorHandler())
opener2.add_handler(urllib2.HTTPErrorProcessor())
elif options.elog == "shyft":
print "Adding SHyFT (JHU) specific post data."
#form.add_field('exp','top')
form.add_field('Type', 'Progress')
form.add_field('Category', 'SHyFT')
form.add_field('Subject', 'Automatic plots posting')
elogUrl = "http://hep.pha.jhu.edu:8080/top/"
def _is_cache_same(self, local_file, remote_url):
"""Checks if the local cache version and the upstream
version is the same or not. If they are the same,
returns True; else False.
"""
with self.fetch_lock:
if not os.path.exists(local_file):
logging.debug(
"No local file cached, will fetch {0}".format(remote_url))
return False
last_checked = self.fetch_last_checked.get(remote_url, 0)
now = time.time()
if now - last_checked <= self.fetch_timeout:
logging.debug(
"Checked for fresh version of '%s' just %f seconds ago. "
"Will wait %f seconds before checking again.", remote_url,
now - last_checked,
self.fetch_timeout - now + last_checked)
return True
opener = urllib.OpenerDirector()
opener.add_handler(urllib.HTTPHandler())
opener.add_handler(urllib.HTTPSHandler())
opener.add_handler(urllib.HTTPDefaultErrorHandler())
# Extra for handling redirects
opener.add_handler(urllib.HTTPErrorProcessor())
opener.add_handler(urllib.HTTPRedirectHandler())
# Add the header
opener.addheaders = self.hdr2
# Grab the header
try:
res = opener.open(CVEFeedManager.HeadRequest(remote_url))
headers = self._parse_http_headers(res.info())
res.close()
remote_ts = headers['last-modified']
except urllib.HTTPError as http_error:
logging.debug(
"Cannot send HTTP HEAD request to get \"last-modified\" "
"attribute of remote content file.\n{0} - {1}".format(
http_error.code, http_error.reason))
return False
except KeyError:
self._print_no_last_modified_warning(remote_url)
return False
self.fetch_last_checked[remote_url] = time.time()
# The remote's datetime
remote_dt = datetime.datetime.strptime(remote_ts,
self.remote_pattern)
# Get the locals datetime from the file's mtime, converted to UTC
local_dt = datetime.datetime.utcfromtimestamp(
os.stat(local_file).st_mtime)
# Giving a two second comfort zone
# Else we declare they are different
if (remote_dt - local_dt).seconds > 2:
logging.info("Had a local version of {0} "
"but it wasn't new enough".format(local_file))
return False
logging.debug("File {0} is same as upstream".format(local_file))
return True
safe_url_opener = urlreq.build_opener(URLSafetyHandler())
if __name__ == '__main__':
import itertools
good_urls = [
'http://www.w3.org/index.html', 'https://w3.org:8080/Overview.html',
'ftp://ftp.w3.org'
]
bad_urls = [
'file:///etc/passwd', 'rsync://w3.org/', 'http://www.w3.org:22/',
'http://localhost/server-status',
'http://localhost:8001/2012/pyRdfa/Overview.html'
]
test_opener = urlreq.OpenerDirector()
test_opener.add_handler(URLSafetyHandler())
check_funcs = [check_url_safety, test_opener.open]
def is_check_passed(call, *args, **kwargs):
try:
call(*args, **kwargs)
except UnsupportedResourceError:
return False
return True
for host in [
'127.0.0.1', '127.254.1.2', '10.1.2.3', '10.254.4.5', '172.16.1.2',
'172.31.4.5', '192.168.0.1', '192.168.254.5', 'fe80::1',
'fe80:ffff::ffff', 'localhost', 'ip6-localhost'
]:
def twitterreq(url, method, parameters):
"""
Send twitter URL request
Utility function used by the others in this package
Note: calls a function twitter_credentials() contained in
a file named twitter_credentials.py which must be provided as follows:
api_key = " your credentials "
api_secret = " your credentials "
access_token_key = " your credentials "
access_token_secret = " your credentials "
return (api_key,api_secret,access_token_key,access_token_secret)
This function is based on a shell provided by
Bill Howe
University of Washington
for the Coursera course Introduction to Data Science
Spring/Summer 2014
(which I HIGHLY recommend)
"""
import oauth2 as oauth
import urllib2 as urllib
# this is a private function containing my Twitter credentials
from twitter_credentials import twitter_credentials
api_key, api_secret, access_token_key, access_token_secret = twitter_credentials(
)
_debug = 0
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
http_method = "GET"
http_handler = urllib.HTTPHandler(debuglevel=_debug)
https_handler = urllib.HTTPSHandler(debuglevel=_debug)
'''
Construct, sign, and open a twitter request
using the hard-coded credentials above.
'''
req = oauth.Request.from_consumer_and_token(oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
headers = req.to_header()
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
opener.add_handler(https_handler)
response = opener.open(url, encoded_post_data)
return response
def do_check(url):
"""
This functinality is based on Filippo's Valsorda script [1]
which uses HEAD requests (with fallback in case of 405)
to follow the redirect path up to the real URL.
---
[1] https://gist.github.com/FiloSottile/2077115
"""
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
class HEADRedirectHandler(urllib2.HTTPRedirectHandler):
"""
Subclass the HTTPRedirectHandler to make it use our
HeadRequest also on the redirected URL
"""
def redirect_request(self, req, fp, code, msg, headers, redirected_url):
if code in (301, 302, 303, 307):
redirected_url = redirected_url.replace(' ', '%20')
newheaders = dict((k,v) for k,v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
warn_msg = "Got a " + str(code) + " redirection (" + redirected_url + ")."
print settings.print_warning_msg(warn_msg)
return HeadRequest(redirected_url,
headers = newheaders,
origin_req_host = req.get_origin_req_host(),
unverifiable = True
)
else:
err_msg = str(urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)).replace(": "," (")
print settings.print_critical_msg(err_msg + ").")
raise SystemExit()
class HTTPMethodFallback(urllib2.BaseHandler):
"""
Fallback to GET if HEAD is not allowed (405 HTTP error)
"""
def http_error_405(self, req, fp, code, msg, headers):
fp.read()
fp.close()
newheaders = dict((k,v) for k,v in req.headers.items() if k.lower() not in ("content-length", "content-type"))
return self.parent.open(urllib2.Request(req.get_full_url(),
headers = newheaders,
origin_req_host = req.get_origin_req_host(),
unverifiable = True)
)
# Build our opener
opener = urllib2.OpenerDirector()
for handler in [urllib2.HTTPHandler,
urllib2.HTTPDefaultErrorHandler,
HTTPMethodFallback,
HEADRedirectHandler,
urllib2.HTTPErrorProcessor,
urllib2.HTTPSHandler]:
opener.add_handler(handler())
try:
response = opener.open(HeadRequest(url))
redirected_url = response.geturl()
if redirected_url != url:
while True:
if not menu.options.batch:
question_msg = "Do you want to follow the identified redirection? [Y/n] > "
sys.stdout.write(settings.print_question_msg(question_msg))
redirection_option = sys.stdin.readline().replace("\n","").lower()
else:
redirection_option = ""
if len(redirection_option) == 0 or redirection_option in settings.CHOICE_YES:
if menu.options.batch:
info_msg = "Following redirection to '" + redirected_url + "'. "
print settings.print_info_msg(info_msg)
return redirected_url
elif redirection_option in settings.CHOICE_NO:
return url
elif redirection_option in settings.CHOICE_QUIT:
sys.exit(0)
else:
err_msg = "'" + redirection_option + "' is not a valid answer."
print settings.print_error_msg(err_msg)
pass
else:
return url
except urllib2.HTTPError, err:
err_msg = str(err).replace(": "," (")
print settings.print_critical_msg(err_msg + ").")
raise SystemExit()
def sentiment(responses_dict):
data = []
terms = {}
intensifier_terms = []
negation_terms = []
positive_tweets = []
negative_tweets = []
neutral_tweets = []
pos_word_set = set()
neg_word_set = set()
for line in responses_dict:
datadict = {}
if "text" in line.keys():
datadict["text"] = line["text"]
pass
if "user" in line.keys():
datadict["user"] = line["user"]["screen_name"]
data.append(datadict)
continue
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
response = opener.open(
"http://twittersentiment.s3.amazonaws.com/sentiment3.txt")
filedata = StringIO(response.read())
for sent_line in filedata:
term, score = sent_line.split(
"\t") # The file is tab-delimited. "\t" means "tab character"
terms[term] = int(score) # Convert the score to an integer
continue
intensifier_response = opener.open(
"http://twittersentiment.s3.amazonaws.com/intensifiers.txt")
intensifier_data = StringIO(intensifier_response.read())
for int_line in intensifier_data:
int_term = int_line.strip()
intensifier_terms.append(int_term)
negation_response = opener.open(
"http://twittersentiment.s3.amazonaws.com/negation1.txt")
negation_data = StringIO(negation_response.read())
for neg_line in negation_data:
neg_term = neg_line.strip()
negation_terms.append(neg_term)
for tweet in data:
if "text" in tweet.keys():
tweet_split = tweet["text"].split()
score = 0
for i, word in enumerate(tweet_split):
new_score, i_term, n_term = rec_unicode_check(
0, i, word, terms, tweet_split, intensifier_terms,
negation_terms)
if new_score > 0:
if i_term != "" or n_term != "":
pos_word_set.add(i_term + n_term + " " + word.lower())
else:
pos_word_set.add(word.lower())
elif new_score < 0:
if i_term != "" or n_term != "":
neg_word_set.add(i_term + n_term + " " + word.lower())
else:
neg_word_set.add(word.lower())
score += new_score
continue
print pos_word_set
print neg_word_set
tweet["score"] = score
if score > 0:
positive_tweets.append(tweet)
elif score < 0:
negative_tweets.append(tweet)
else:
neutral_tweets.append(tweet)
final_list = sorted(data, key=lambda k: k['score'])
d3data = [len(positive_tweets), len(neutral_tweets), len(negative_tweets)]
return final_list, d3data, pos_word_set, neg_word_set
def _make_opener(url):
opener = urllib2.OpenerDirector()
for handler_class in _DEFAULT_HANDLERS:
opener.add_handler(handler_class())
opener.add_handler(VerifiedHTTPSHandler())
return opener
def setUp(self):
self.http_mock = Mock(side_effect=[])
opener = urllib2.OpenerDirector()
opener.add_handler(TestHandler(self.http_mock))
urllib2.install_opener(opener)
def check_http_traffic(request):
# Delay in seconds between each HTTP request
time.sleep(int(settings.DELAY))
class do_connection(httplib.HTTPConnection, httplib.HTTPSConnection):
"""
Checking the HTTP / HTTPS requests.
"""
def request(self, method, url, body, headers):
info_msg = "The provided HTTP request headers: "
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_info_msg(info_msg)
if menu.options.traffic_file:
logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37)
header = method + " " + url
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
for item in headers.items():
header = item[0] + ": " + item[1]
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
if body:
header = body
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
if menu.options.traffic_file:
logs.log_traffic("\n\n")
if settings.PROXY_PROTOCOL == 'https':
httplib.HTTPSConnection.request(self, method, url, body,
headers)
else:
httplib.HTTPConnection.request(self, method, url, body,
headers)
class connection_handler(urllib2.HTTPHandler, urllib2.HTTPSHandler):
if settings.PROXY_PROTOCOL == 'https':
def https_open(self, req):
return self.do_open(do_connection, req)
else:
def http_open(self, req):
return self.do_open(do_connection, req)
if settings.REVERSE_TCP == False and settings.BIND_TCP == False:
opener = urllib2.OpenerDirector()
opener.add_handler(connection_handler())
try:
opener.open(request)
except:
pass
response = urllib2.urlopen(request)
# Check the HTTP response headers.
http_response(response.info())
# Check the HTTP response content.
http_response_content(response.read())
return response
def _on_urllib2_urlopen(self,
url,
data=None,
timeout=None,
cafile=None,
capath=None,
cadefault=False,
context=None):
request = Request(url.get_method(), url.get_full_url(), data,
url.headers, url.get_type(), url.get_host(),
url.port)
match = self._find_match(request)
resp_callback = self.response_callback
if self.allow_external_requests:
logger.info('request.allowed-external',
extra={
'url': request.url,
})
return _real_urllib2_urlopen(url)
if match is None:
if request.url.startswith(self.passthru_prefixes):
logger.info('request.allowed-passthru',
extra={
'url': request.url,
})
return _real_urllib2_urlopen(url)
response = self._on_no_match(request, resp_callback)
raise response
class MockHTTPSConnection(HTTPSConnection):
def getresponse(buffering=False):
response = match.get_response(request)
response.msg = ''
return response
class MockHTTPConnection(HTTPConnection):
def getresponse(buffering=False):
response = match.get_response(request)
response.msg = ''
return response
class MockHTTPSHandler(urllib2.AbstractHTTPHandler):
def __init__(self, debuglevel=0, context=None):
urllib2.AbstractHTTPHandler.__init__(self, debuglevel)
self._context = context
def https_open(self, req):
return self.do_open(MockHTTPSConnection,
req,
context=self._context)
class MockHTTPHandler(urllib2.AbstractHTTPHandler):
def http_open(self, req):
return self.do_open(MockHTTPConnection, req)
response = match.get_response(request)
opener = urllib2.OpenerDirector()
opener.add_handler(MockHTTPHandler())
if cafile or cadefault:
if context is not None:
raise ValueError(
"You can't pass both context and any of cafile, capath, and "
"cadefault")
if not _have_ssl:
raise ValueError('SSL support not available')
context = ssl.create_default_context(
purpose=ssl.Purpose.SERVER_AUTH, cafile=cafile, capath=capath)
https_handler = MockHTTPSHandler(context=context)
opener.add_handler(https_handler)
elif context:
https_handler = MockHTTPSHandler(context=context)
opener = opener.add_handler(https_handler)
elif hasattr(httplib, 'HTTPS'):
opener.add_handler(MockHTTPSHandler())
self._calls.add(request, response)
return opener.open(url, data, timeout)
api_secret = "**"
access_token_key = "*-**"
access_token_secret = "**"
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
## Send request to url
url = "https://stream.twitter.com/1.1/statuses/sample.json"
req = oauth.Request.from_consumer_and_token(oauth_consumer, token=oauth_token,
http_method="GET", http_url=url)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
url = req.to_url()
## Open the connection
opener = urllib.OpenerDirector()
http_handler = urllib.HTTPHandler(debuglevel=0)
https_handler = urllib.HTTPSHandler(debuglevel=0)
opener.add_handler(http_handler)
opener.add_handler(https_handler)
## Get the response
response = opener.open(url)
## Write response to file
f = open('tweets.json', 'w')
i = 0
for line in response:
entry = json.loads(line)
## Keep English tweets only
if u'lang' in entry and entry[u'lang'] == 'en' and u'text' in entry:
"""A password manager that grabs credentials from upwards in
its call stack.
This is for cooperation with urlopenRemote, which defines a name
_temp_credentials. If this is non-None, it's supposed to be
a pair of user password presented to *any* realm. This means
that, at least with http basic auth, password stealing is
almost trivial.
"""
def find_user_password(self, realm, authuri):
creds = codetricks.stealVar("_temp_credentials")
if creds is not None:
return creds
_restrictedURLOpener = urllib2.OpenerDirector()
_restrictedURLOpener.add_handler(urllib2.HTTPRedirectHandler())
_restrictedURLOpener.add_handler(urllib2.HTTPHandler())
_restrictedURLOpener.add_handler(urllib2.HTTPSHandler())
_restrictedURLOpener.add_handler(urllib2.HTTPErrorProcessor())
_restrictedURLOpener.add_handler(
urllib2.HTTPBasicAuthHandler(_UrlopenRemotePasswordMgr()))
_restrictedURLOpener.add_handler(urllib2.FTPHandler())
_restrictedURLOpener.add_handler(urllib2.UnknownHandler())
_restrictedURLOpener.addheaders = [("user-agent", "GAVO DaCHS HTTP client")]
def urlopenRemote(url, data=None, creds=(None, None)):
"""works like urllib2.urlopen, except only http, https, and ftp URLs
are handled.
def check_http_traffic(request):
# Delay in seconds between each HTTP request
time.sleep(int(settings.DELAY))
class do_connection(httplib.HTTPConnection, httplib.HTTPSConnection):
"""
Checking the HTTP / HTTPS requests.
"""
def request(self, method, url, body, headers):
info_msg = "The provided HTTP request headers: "
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_info_msg(info_msg)
if menu.options.traffic_file:
logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37)
header = method + " " + url
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
for item in headers.items():
header = item[0] + ": " + item[1]
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
if body:
header = body
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
if menu.options.traffic_file:
logs.log_traffic("\n\n")
if settings.PROXY_PROTOCOL == 'https':
httplib.HTTPSConnection.request(self, method, url, body,
headers)
else:
httplib.HTTPConnection.request(self, method, url, body,
headers)
class connection_handler(urllib2.HTTPHandler, urllib2.HTTPSHandler):
if settings.PROXY_PROTOCOL == 'https':
def https_open(self, req):
return self.do_open(do_connection, req)
else:
def http_open(self, req):
return self.do_open(do_connection, req)
if settings.REVERSE_TCP == False and settings.BIND_TCP == False:
opener = urllib2.OpenerDirector()
opener.add_handler(connection_handler())
response = False
current_attempt = 0
while not response and current_attempt <= settings.MAX_RETRIES:
try:
opener.open(request)
response = True
if settings.VERBOSITY_LEVEL < 2:
if current_attempt != 0:
info_msg = "Checking connection to the target URL... "
sys.stdout.write(settings.print_info_msg(info_msg))
sys.stdout.flush()
if settings.INIT_TEST == True:
print "[ " + Fore.GREEN + "SUCCEED" + Style.RESET_ALL + " ]"
if not settings.CHECK_INTERNET:
settings.INIT_TEST = False
except urllib2.URLError, err_msg:
if menu.options.check_internet:
if settings.VERBOSITY_LEVEL < 2:
print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]"
break
if current_attempt == 0:
if settings.VERBOSITY_LEVEL < 2:
print "[ " + Fore.RED + "FAILED" + Style.RESET_ALL + " ]"
error_msg = str(err_msg.args[0]).split("] ")[1] + ". "
error_msg += "Please wait while retring the request(s)."
print settings.print_critical_msg(error_msg)
warn_msg = "In case the provided target URL is valid, try to rerun with"
warn_msg += " the switch '--random-agent' and/or proxy switch."
print settings.print_warning_msg(warn_msg)
if settings.VERBOSITY_LEVEL >= 2 or current_attempt == 1:
#info_msg = str(err_msg.args[0]).split("] ")[1] + ". "
info_msg = "Please wait while retring the request(s)."
print settings.print_info_msg(info_msg)
current_attempt = current_attempt + 1
time.sleep(3)
pass
dictWriter.writeheader()
dictWriter.writerows(dict)
# print dict
if __name__ == "__main__":
# main_1()
import urllib2
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
manager.add_password(None, 'https://app.streamsend.com/emails', 'login',
'key')
handler = urllib2.HTTPBasicAuthHandler(manager)
director = urllib2.OpenerDirector()
director.add_handler(handler)
req = urllib2.Request('https://app.streamsend.com/emails',
headers={'Accept': 'application/xml'})
result = director.open(req)
# result.read() will contain the data
# result.info() will contain the HTTP headers
# To get say the content-length header
length = result.info()['Content-Length']
"""
while ( True ) :
if len(sys.argv) > 1:
def do_check(url):
"""
This functinality is based on Filippo's Valsorda script [1].
---
[1] https://gist.github.com/FiloSottile/2077115
"""
class Request(urllib2.Request):
def get_method(self):
return "GET"
class RedirectHandler(urllib2.HTTPRedirectHandler):
"""
Subclass the HTTPRedirectHandler to make it use our
Request also on the redirected URL
"""
def redirect_request(self, req, fp, code, msg, headers,
redirected_url):
if code in (301, 302, 303, 307):
redirected_url = redirected_url.replace(' ', '%20')
newheaders = dict(
(k, v) for k, v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
warn_msg = "Got a " + str(
code) + " redirection (" + redirected_url + ")."
print settings.print_warning_msg(warn_msg)
return Request(redirected_url,
headers=newheaders,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
else:
err_msg = str(
urllib2.HTTPError(req.get_full_url(), code, msg, headers,
fp)).replace(": ", " (")
print settings.print_critical_msg(err_msg + ").")
raise SystemExit()
class HTTPMethodFallback(urllib2.BaseHandler):
"""
"""
def http_error_405(self, req, fp, code, msg, headers):
fp.read()
fp.close()
newheaders = dict(
(k, v) for k, v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
return self.parent.open(
urllib2.Request(req.get_full_url(),
headers=newheaders,
origin_req_host=req.get_origin_req_host(),
unverifiable=True))
# Build our opener
opener = urllib2.OpenerDirector()
# Check if defined any Host HTTP header.
if menu.options.host and settings.HOST_INJECTION == False:
opener.addheaders.append(('Host', menu.options.host))
# Check if defined any User-Agent HTTP header.
if menu.options.agent:
opener.addheaders.append(('User-Agent', menu.options.agent))
# Check if defined any Referer HTTP header.
if menu.options.referer and settings.REFERER_INJECTION == False:
opener.addheaders.append(('Referer', menu.options.referer))
# Check if defined any Cookie HTTP header.
if menu.options.cookie and settings.COOKIE_INJECTION == False:
opener.addheaders.append(('Cookie', menu.options.cookie))
for handler in [
urllib2.HTTPHandler, HTTPMethodFallback, RedirectHandler,
urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler
]:
opener.add_handler(handler())
try:
response = opener.open(Request(url))
redirected_url = response.geturl()
if redirected_url != url:
while True:
if not menu.options.batch:
question_msg = "Do you want to follow the identified redirection? [Y/n] > "
sys.stdout.write(settings.print_question_msg(question_msg))
redirection_option = sys.stdin.readline().replace(
"\n", "").lower()
else:
redirection_option = ""
if len(redirection_option
) == 0 or redirection_option in settings.CHOICE_YES:
if menu.options.batch:
info_msg = "Following redirection to '" + redirected_url + "'. "
print settings.print_info_msg(info_msg)
return redirected_url
elif redirection_option in settings.CHOICE_NO:
return url
elif redirection_option in settings.CHOICE_QUIT:
raise SystemExit()
else:
err_msg = "'" + redirection_option + "' is not a valid answer."
print settings.print_error_msg(err_msg)
pass
else:
return url
except AttributeError:
pass
# Raise exception due to ValueError.
except ValueError, err:
err_msg = str(err).replace(": ", " (")
print settings.print_critical_msg(err_msg + ").")
raise SystemExit()
def make_opener(self, address, headers=None):
scheme = None
proxy_host = None
proxy_password_manager = None
http_password_manager = OptionalPasswordManager()
password_managers = []
if self.proxy == 'wpad':
proxy = get_proxy_for_address(address)
if proxy:
proxy = proxy[0]
else:
proxy = None
else:
proxy = self.proxy
if not proxy or proxy[0] == 'DIRECT' or self._is_direct(address):
handlers = [
StreamingHTTPHandler,
StreamingHTTPSHandler(context=self.ctx),
TCPReaderHandler(context=self.ctx)
]
else:
scheme, host, user, password = proxy
scheme = PROXY_TYPES[scheme]
port = DEFAULT_PORTS[scheme]
if ':' in host:
host, maybe_port = host.split(':')
try:
port = int(maybe_port)
except ValueError:
pass
proxy_host = host + ':' + str(port)
sockshandler = SocksiPyHandler(
scheme,
host,
port,
user or None,
password or None,
context=self.ctx if self.noverify else None)
handlers = []
if scheme == PROXY_SCHEME_HTTP:
http_proxy = proxy_host
handlers.append(
urllib2.ProxyHandler({'http': 'http://' + http_proxy}))
proxy_password_manager = ProxyPasswordManager(
'http', host, port, user, password)
for handler_klass in (ProxyAuthHandler,
urllib2.ProxyBasicAuthHandler,
urllib2.ProxyDigestAuthHandler):
if handler_klass is None:
continue
instance = handler_klass(proxy_password_manager)
if hasattr(instance, 'set_logger'):
instance.set_logger(logger)
handlers.append(instance)
password_managers.append(proxy_password_manager)
handlers.append(StreamingHTTPHandler)
handlers.append(sockshandler)
if self.follow_redirects:
handlers.append(urllib2.HTTPRedirectHandler)
else:
handlers.append(NoRedirects)
handlers.append(UDPReaderHandler)
for handler_klass in (urllib2.HTTPBasicAuthHandler,
urllib2.HTTPDigestAuthHandler, HTTPAuthHandler):
if handler_klass is None:
continue
instance = handler_klass(http_password_manager)
if hasattr(instance, 'set_logger'):
instance.set_logger(logger)
handlers.append(instance)
password_managers.append(http_password_manager)
context = HTTPContext.get_default()
handlers.append(context)
handlers.append(urllib2.HTTPDefaultErrorHandler)
handlers.append(urllib2.HTTPErrorProcessor)
opener = urllib2.OpenerDirector()
for h in handlers:
if isinstance(h, (types.ClassType, type)):
h = h()
opener.add_handler(h)
filter_headers = set()
if headers:
if isinstance(headers, dict):
filter_headers = set(headers.keys())
else:
filter_headers = set(x for x, _ in headers)
if isinstance(self.headers, dict):
opener.addheaders = [(x, y) for x, y in self.headers.iteritems()
if x not in filter_headers]
else:
opener.addheaders = self.headers
if headers:
if isinstance(headers, dict):
opener.addheaders.extend([
(x, y) for x, y in self.headers.iteritems()
])
else:
opener.addheaders.extend(headers)
return opener, scheme, proxy_host, password_managers, context
class WizardTool(BaseTool):
""" WizardTool is able to generate custom business templates. """
id = 'portal_wizard'
meta_type = 'ERP5 Wizard Tool'
portal_type = 'Wizard Tool'
isPortalContent = ConstantGetter('isPortalContent', value=True)
property_sheets = ()
security = ClassSecurityInfo()
security.declareProtected(Permissions.ManagePortal, 'manage_overview')
manage_overview = DTMLFile('explainWizardTool', _dtmldir)
# Stop traversing a concatenated path after the proxy method.
def __before_publishing_traverse__(self, self2, request):
path = request['TraversalRequestNameStack']
if path and path[-1] == 'proxy':
subpath = path[:-1]
subpath.reverse()
request.set('traverse_subpath', subpath)
# initialize our root proxy URL which we use for a referer
global referer
path[:-1] = []
if referer is None:
referer = '%s/portal_wizard/proxy/%s/view' % (
self.getPortalObject().absolute_url(), '/'.join(
subpath[:3]))
def _getProxyURL(self, subpath='', query=''):
# Helper method to construct an URL appropriate for proxying a request.
# This makes sure that URLs generated by absolute_url at a remote site
# will be always towards the proxy method again.
#
# Note that the code assumes that VirtualHostBase is visible. The setting
# of a front-end server must allow this.
#
# This should generate an URL like this:
#
# http://remotehost:9080/VirtualHostBase/http/localhost:8080/VirtualHostRoot/_vh_erp5/_vh_portal_wizard/_vh_proxy/erp5/person_module/2
part_list = []
server_url = self.getServerUrl().rstrip('/')
part_list.append(server_url)
part_list.append('VirtualHostBase')
portal_url = self.getPortalObject().absolute_url()
scheme, rest = urllib.splittype(portal_url)
addr, path = urllib.splithost(rest)
host, port = urllib.splitnport(addr, scheme == 'http' and 80 or 443)
part_list.append(scheme)
part_list.append('%s:%s' % (host, port))
part_list.append('VirtualHostRoot')
method_path = self.absolute_url_path() + '/proxy'
part_list.extend(('_vh_' + p for p in method_path.split('/') if p))
server_root = self.getServerRoot().strip('/')
if isinstance(subpath, (list, tuple)):
subpath = '/'.join(subpath)
if not subpath.startswith(server_root):
part_list.append(server_root)
part_list.append(subpath)
url = '/'.join((p for p in part_list if p))
if query:
url = url + '?' + query
return url
def _getSubsribedUserAndPassword(self):
"""Retrieve the username and password for the subscription from
the system."""
user = CachingMethod(self.getExpressConfigurationPreference,
'WizardTool_preferred_express_user_id',
cache_factory='erp5_content_long')(
'preferred_express_user_id', '')
pw = CachingMethod(self.getExpressConfigurationPreference,
'WizardTool_preferred_express_password',
cache_factory='erp5_content_long')(
'preferred_express_password', '')
return (user, pw)
# This is a custom opener director for not handling redirections
# and errors automatically. This is necessary because the proxy
# should pass all results to a client as they are.
simple_opener_director = urllib2.OpenerDirector()
for name in ('ProxyHandler', 'UnknownHandler', \
'HTTPHandler', 'FTPHandler',
'FileHandler', 'HTTPSHandler',):
handler = getattr(urllib2, name, None)
if handler is not None:
simple_opener_director.add_handler(handler())
# add cookie support
simple_opener_director.add_handler(urllib2.HTTPCookieProcessor(cookiejar))
security.declareProtected(Permissions.View, 'proxy')
def proxy(self, **kw):
"""Proxy a request to a server."""
global cookiejar, referer, last_loggedin_user_and_password
if self.REQUEST['REQUEST_METHOD'] != 'GET':
# XXX this depends on the internal of HTTPRequest.
pos = self.REQUEST.stdin.tell()
self.REQUEST.stdin.seek(0)
# XXX if filesize is too big, this might cause a problem.
data = self.REQUEST.stdin.read()
self.REQUEST.stdin.seek(pos)
else:
data = None
content_type = self.REQUEST.get_header('content-type')
# XXX if ":method" trick is used, then remove it from subpath.
if self.REQUEST.traverse_subpath:
if data is not None:
user_input = data
else:
user_input = self.REQUEST.QUERY_STRING
if user_input:
mark = ':method'
content_type_value = None
content_type_dict = None
if content_type:
content_type_value, content_type_dict = cgi.parse_header(
content_type)
if content_type_value == 'multipart/form-data':
fp = StringIO(user_input)
user_input_dict = cgi.parse_multipart(
fp, content_type_dict)
else:
user_input_dict = cgi.parse_qs(user_input)
for i in user_input_dict:
if i.endswith(mark):
method_name = i[:-len(mark)]
method_path = method_name.split('/')
if self.REQUEST.traverse_subpath[
-len(method_path):] == method_path:
del self.REQUEST.traverse_subpath[-len(method_path
):]
break
url = self._getProxyURL(self.REQUEST.traverse_subpath,
self.REQUEST['QUERY_STRING'])
# XXX this will send the password unconditionally!
# I hope https will be good enough.
header_dict = {}
user_and_password = self._getSubsribedUserAndPassword()
if (len(user_and_password) == 2 and user_and_password[0]
and user_and_password[1]):
if user_and_password != last_loggedin_user_and_password:
# credentials changed we need to renew __ac cookie from server as well
cookiejar.clear()
# try login to server only once using cookie method
if not _isUserAcknowledged(cookiejar):
server_url = self.getServerUrl()
f = _getAcCookieFromServer('%s/WebSite_login' % server_url,
self.simple_opener_director,
cookiejar, user_and_password[0],
user_and_password[1])
# if server doesn't support cookie authentication try basic
# authentication
if not _isUserAcknowledged(cookiejar):
auth = 'Basic %s' % base64.standard_b64encode(
'%s:%s' % user_and_password)
header_dict['Authorization'] = auth
# save last credentials we passed to server
last_loggedin_user_and_password = user_and_password
if content_type:
header_dict['Content-Type'] = content_type
# send locally saved cookies to remote web server
if not header_dict.has_key('Cookie'):
header_dict['Cookie'] = ''
for cookie in cookiejar:
# unconditionally send all cookies (no matter if expired or not) as URL
# is always the same
header_dict['Cookie'] += '%s=%s;' % (cookie.name, cookie.value)
# include cookies from local browser (like show/hide tabs) which are set
# directly by client JavaScript code (i.e. not sent from server)
for cookie_name, cookie_value in self.REQUEST.cookies.items():
header_dict['Cookie'] += '%s=%s;' % (cookie_name, cookie_value)
# add HTTP referer (especially useful in Localizer when changing language)
header_dict['REFERER'] = self.REQUEST.get('HTTP_REFERER',
None) or referer
request = urllib2.Request(url, data, header_dict)
f = self.simple_opener_director.open(request)
try:
data = f.read()
metadata = f.info()
response = self.REQUEST.RESPONSE
if f.code > 300 and f.code < 400:
# adjust return url which my contain proxy URLs as arguments
location = metadata.getheader('location')
if location is not None:
parsed_url = list(urlparse(location))
local_site_url_prefix = urllib.quote(
'%s/portal_wizard/proxy' %
self.getPortalObject().absolute_url())
remote_url_parsed = urlparse(self.getServerUrl())
remote_site_url_prefix = '%s://%s/kb' % (
remote_url_parsed[0], remote_url_parsed[1])
# fix arguments for returned location URL
parsed_url[4] = parsed_url[4].replace(
local_site_url_prefix, remote_site_url_prefix)
response['location'] = urlunparse(parsed_url)
response.setStatus(f.code, f.msg)
response.setHeader('content-type',
metadata.getheader('content-type'))
# FIXME this list should be confirmed with the RFC 2616.
for k in ('uri', 'cache-control', 'last-modified', 'etag',
'if-matched', 'if-none-match', 'if-range',
'content-language', 'content-range'
'content-location', 'content-md5', 'expires',
'content-encoding', 'vary', 'pragma',
'content-disposition', 'content-length', 'age'):
if k in metadata:
response.setHeader(k, metadata.getheader(k))
return data
finally:
f.close()
def _getRemoteWitchTool(self, url, user_name=None, password=None):
"""
Return remote portal_witch tool interface.
"""
handle = self.getPortalObject().portal_web_services.connect(
url=url,
user_name=user_name,
password=password,
transport='xml-rpc')
return handle.portal_witch
def callRemoteProxyMethod(self,
distant_method,
server_url=None,
use_cache=1,
ignore_exceptions=1,
**kw):
""" Call proxy method on server. """
configurator_user_preferred_language = self\
.getConfiguratorUserPreferredLanguage()
def wrapper(distant_method, **kw):
return self._callRemoteMethod(distant_method,
use_proxy=1,
ignore_exceptions=ignore_exceptions,
**kw)['data']
if use_cache:
wrapper = CachingMethod(
wrapper,
id='callRemoteProxyMethod_%s_%s' %
(distant_method, configurator_user_preferred_language),
cache_factory='erp5_ui_medium')
rc = wrapper(distant_method, **kw)
return rc
def _callRemoteMethod(self,
distant_method,
server_url=None,
use_proxy=0,
ignore_exceptions=1,
**kw):
""" Call remote method on server and get result. """
result_call = GeneratorCall()
user_name = None
password = None
if server_url is None:
# calculate it
server_url = self.getServerUrl() + self.getServerRoot()
# include authentication if possible
user_and_password = self._getSubsribedUserAndPassword()
if (len(user_and_password) == 2 and user_and_password[0]
and user_and_password[1]):
user_name, password = user_and_password
witch_tool = self._getRemoteWitchTool(server_url, user_name, password)
parameter_dict = self.REQUEST.form.copy()
if use_proxy:
# add remote method arguments
parameter_dict['method_id'] = distant_method
parameter_dict['method_kw'] = kw
distant_method = 'proxyMethodHandler'
## add client arguments
self._updateParameterDictWithServerInfo(parameter_dict)
## handle file upload
self._updateParameterDictWithFileUpload(parameter_dict)
message = None
## call remote method
try:
method = getattr(witch_tool, distant_method)
html = method(parameter_dict)
except socket.error, message:
html = _generateErrorXML("""Cannot contact the server: %s.
Please check your network settings.""" %
server_url)
zLOG.LOG('Wizard Tool socket error', zLOG.ERROR, message)
result_call.update({
"command": "show",
"data": html,
"next": None,
"previous": None
})
except xmlrpclib.ProtocolError, message:
html = _generateErrorXML("""The server %s refused to reply.
Please contact [email protected]""" %
server_url)
zLOG.LOG('Wizard Tool xmlrpc protocol error', zLOG.ERROR, message)
result_call.update({
"command": "show",
"data": html,
"next": None,
"previous": None
})
def check_http_traffic(request):
# Delay in seconds between each HTTP request
time.sleep(int(settings.DELAY))
class do_connection(httplib.HTTPConnection, httplib.HTTPSConnection):
"""
Checking the HTTP / HTTPS requests.
"""
def request(self, method, url, body, headers):
info_msg = "The provided HTTP request headers: "
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_info_msg(info_msg)
if menu.options.traffic_file:
logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37)
header = method + " " + url
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
for item in headers.items():
header = item[0] + ": " + item[1]
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
if body:
header = body
if settings.VERBOSITY_LEVEL >= 2:
print settings.print_traffic(header)
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
if menu.options.traffic_file:
logs.log_traffic("\n\n")
if settings.PROXY_PROTOCOL == 'https':
httplib.HTTPSConnection.request(self, method, url, body,
headers)
else:
httplib.HTTPConnection.request(self, method, url, body,
headers)
class connection_handler(urllib2.HTTPHandler, urllib2.HTTPSHandler):
if settings.PROXY_PROTOCOL == 'https':
def https_open(self, req):
return self.do_open(do_connection, req)
else:
def http_open(self, req):
return self.do_open(do_connection, req)
if settings.REVERSE_TCP == False and settings.BIND_TCP == False:
opener = urllib2.OpenerDirector()
opener.add_handler(connection_handler())
response = False
current_attempt = 0
while not response and current_attempt <= settings.MAX_RETRIES:
try:
opener.open(request)
response = True
except urllib2.URLError, err_msg:
if settings.VERBOSITY_LEVEL >= 2:
info_msg = str(err_msg.args[0]).split("] ")[1] + ", "
info_msg += "please wait while retring the request(s)."
print settings.print_info_msg(info_msg)
current_attempt = current_attempt + 1
time.sleep(3)
pass
