def testMessage(self):
conn = HTTPConnection('localhost:8080')
conn.request("GET", "/message?id=1")
request = conn.getresponse()
result = request.read()
self.assertEqual(request.status, 200)
self.assertGreater(result.find('<div class="messagebox">'), 0)
conn.request("GET", "/message?id=2")
request = conn.getresponse()
result = request.read()
self.assertEqual(request.status, 200)
self.assertGreater(result.find('Please select an email message from the list'), 0)
conn.request("GET", "/message?id=-2")
request = conn.getresponse()
result = request.read()
self.assertEqual(request.status, 200)
self.assertGreater(result.find('Please select an email message from the list'), 0)
conn = HTTPConnection('localhost:8080')
conn.request("GET", "/message?id=1&onlymsg=true")
request = conn.getresponse()
result = request.read()
result.find('The HTML Body')
self.assertEqual(request.status, 200)
self.assertEqual(result, 'The HTML Body')
conn.close()
def send_email(request):
try:
recipients = request.GET["to"].split(",")
url = request.GET["url"]
proto, server, path, query, frag = urlsplit(url)
if query:
path += "?" + query
conn = HTTPConnection(server)
conn.request("GET", path)
try: # Python 2.7+, use buffering of HTTP responses
resp = conn.getresponse(buffering=True)
except TypeError: # Python 2.6 and older
resp = conn.getresponse()
assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason)
rawData = resp.read()
conn.close()
message = MIMEMultipart()
message["Subject"] = "Graphite Image"
message["To"] = ", ".join(recipients)
message["From"] = "composer@%s" % gethostname()
text = MIMEText("Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(), url))
image = MIMEImage(rawData)
image.add_header("Content-Disposition", "attachment", filename="composer_" + strftime("%b%d_%I%M%p.png"))
message.attach(text)
message.attach(image)
s = SMTP(settings.SMTP_SERVER)
s.sendmail("composer@%s" % gethostname(), recipients, message.as_string())
s.quit()
return HttpResponse("OK")
except:
return HttpResponse(format_exc())
def run(self):
if self.file:
print "RUN THE THREAD CLIENT"
dataFile = open(self.file, "r")
cdata = dataFile.read()
conn = HTTPConnection(self.proxy)
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if rdata == cdata:
self.result = True
self.data = rdata
conn.close()
dataFile.close()
print "CONNECTION CLOSE"
else:
conn = HTTPConnection(self.proxy)
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if resp.status == httplib.OK:
self.result = True
conn.close()
def test_get(self):
PORT = TestMemHTTPServer.PORT
server = MemHTTPServer(('localhost', PORT))
server.set_get_output('asdf', 'text/html', 'ASDF')
server.set_get_output('qwer', 'text/plain', 'QWER')
server.server_activate()
client = HTTPConnection('localhost', PORT)
client.connect()
client.request('GET', 'asdf')
server.handle_request()
response = client.getresponse()
self.assertEqual(200, response.status)
self.assertEqual('ASDF', response.read())
self.assertEqual('text/html', response.getheader('Content-type'))
client = HTTPConnection('localhost', PORT)
client.connect()
client.request('GET', 'qwer')
server.handle_request()
response = client.getresponse()
self.assertEqual(200, response.status)
self.assertEqual('QWER', response.read())
self.assertEqual('text/plain', response.getheader('Content-type'))
def _run_server(self, app):
"""Run a wsgi server in a separate thread"""
ip, port = get_free_port()
self.app = app = WSGIApplication(app, (ip, port))
def run():
logger = logging.getLogger("SeleniumWebDriverApp")
def log_message(self, format, *args):
logger.info("%s - - [%s] %s\n" %
(self.address_string(),
self.log_date_time_string(),
format % args))
# monkey patch to redirect request handler logs
WSGIRequestHandler.log_message = log_message
httpd = simple_server.make_server(ip, port, app,
server_class=WSGIServer,
handler_class=WSGIRequestHandler)
httpd.serve_forever()
app.thread = Process(target=run)
app.thread.start()
conn = HTTPConnection(ip, port)
time.sleep(.5)
for i in range(100):
try:
conn.request('GET', '/__application__')
conn.getresponse()
except (socket.error, CannotSendRequest):
time.sleep(.3)
else:
break
def download_default_pages(names, prefix):
from httplib import HTTPConnection
host = 'trac.edgewall.org'
if prefix and not prefix.endswith('/'):
prefix += '/'
conn = HTTPConnection(host)
for name in names:
if name in ('WikiStart', 'SandBox'):
continue
sys.stdout.write('Downloading %s%s' % (prefix, name))
conn.request('GET', '/wiki/%s%s?format=txt' % (prefix, name))
response = conn.getresponse()
content = response.read()
if prefix and (response.status != 200 or not content):
sys.stdout.write(' %s' % name)
conn.request('GET', '/wiki/%s?format=txt' % name)
response = conn.getresponse()
content = response.read()
if response.status == 200 and content:
with open('trac/wiki/default-pages/' + name, 'w') as f:
lines = content.replace('\r\n', '\n').splitlines(True)
f.write(''.join(line for line in lines
if line.strip() != '[[TranslatedPages]]'))
sys.stdout.write('\tdone.\n')
else:
sys.stdout.write('\tmissing or empty.\n')
conn.close()
def fetchData ( server, markets, network ):
marketsData = None
networkData = None
connection = HTTPConnection ( server, timeout=5 )
# markets
connection.request ( 'GET', markets )
r = connection.getresponse ( )
if ( r.status == 200 ):
marketsData = r.read ( )
else:
stderr.write ( 'error: fetchData() got status %d for markets\n' % r.status )
return None
# network
connection.request ( 'GET', network )
r = connection.getresponse ( )
if ( r.status == 200 ):
networkData = r.read ( )
else:
stderr.write ( 'error: fetchData() got status %d for network\n' % r.status )
return None
connection.close ( )
# parse JSON
return ( json.loads(marketsData), json.loads(networkData), '' )
def run(self):
conn = HTTPConnection(self.proxy)
tmpFlag = True
dataFile = open(self.file, "r")
cdata = dataFile.read()
dataFile = open(self.file2, "r")
cdata2 = dataFile.read()
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if rdata != cdata:
tmpFlag = False
if resp.will_close == True:
tmpFlag = False
connHdrs = {"Connection": "close"}
conn.request("GET", self.url2, headers=connHdrs)
resp = conn.getresponse()
rdata2 = resp.read()
if rdata2 != cdata2:
tmpFlag = False
if resp.will_close == False:
tmpFlag = False
if tmpFlag == True:
self.result = True
conn.close()
dataFile.close()
def run(self):
if self.file:
dataFile = open(self.file, "r")
cdata = dataFile.read()
conn = HTTPConnection(self.proxy)
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if rdata == cdata:
self.result = True
self.data = rdata
conn.close()
dataFile.close()
else:
conn = HTTPConnection(self.proxy)
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if resp.status == httplib.OK:
self.result = True
conn.close()
def run(self):
ts = time()
# print "Health check on", self.nodename
conn = Conn(self.nodename, PORT, timeout=30)
eltime = -1
error = ""
try:
conn.request("GET", "/esgf-nm/health-check-api?from=" + localhostname + "&forward=" + str(self.fwdcheck))
resp = conn.getresponse()
if resp.status == 500:
print "500 error"
self.logger.error(resp.read())
eltime = time() - ts
self.handle_resp(resp)
except Exception as e:
error = "connectivity problem"
print e
self.eltime = eltime
nodemap_instance = get_instance()
node_list = nodemap_instance.get_supernode_list()
if not self.fwdcheck:
if not self.checkarr is None:
self.checkarr.append(self.nodename + "=" + str(eltime))
if (self.first):
if len(node_list) > len(self.checkarr) + 2:
sleep(.01)
conn = Conn(self.fromnode, PORT, timeout=30)
url = "/esgf-nm/health-check-rep?from=" + localhostname
for n in self.checkarr:
url = url + "&" + n
error = ""
resp = ""
try:
conn.request("GET", url)
resp = conn.getresponse()
self.handle_resp(resp)
except Exception as e:
error = "connectivity problem"
print e
class HTTPRelayClient:
def __init__(self, target):
# Target comes as protocol://target:port/path
self.target = target
_, host, path = target.split(':')
host = host[2:]
self.path = '/' + path.split('/')[1]
self.session = HTTPConnection(host)
self.lastresult = None
def sendNegotiate(self,negotiateMessage):
#Check if server wants auth
self.session.request('GET', self.path)
res = self.session.getresponse()
res.read()
if res.status != 401:
logging.info('Status code returned: %d. Authentication does not seem required for URL' % res.status)
try:
if 'NTLM' not in res.getheader('WWW-Authenticate'):
logging.error('NTLM Auth not offered by URL, offered protocols: %s' % res.getheader('WWW-Authenticate'))
return False
except KeyError:
logging.error('No authentication requested by the server for url %s' % self.target)
return False
#Negotiate auth
negotiate = base64.b64encode(negotiateMessage)
headers = {'Authorization':'NTLM %s' % negotiate}
self.session.request('GET', self.path ,headers=headers)
res = self.session.getresponse()
res.read()
try:
serverChallenge = base64.b64decode(res.getheader('WWW-Authenticate')[5:])
return serverChallenge
except (IndexError, KeyError):
logging.error('No NTLM challenge returned from server')
def sendAuth(self,authenticateMessageBlob, serverChallenge=None):
#Negotiate auth
auth = base64.b64encode(authenticateMessageBlob)
headers = {'Authorization':'NTLM %s' % auth}
self.session.request('GET', self.path,headers=headers)
res = self.session.getresponse()
if res.status == 401:
return False
else:
logging.info('HTTP server returned error code %d, treating as a succesful login' % res.status)
#Cache this
self.lastresult = res.read()
return True
#SMB Relay server needs this
@staticmethod
def get_encryption_key():
return None
class CommandLineFu(object):
def __init__(self):
self._h = HTTP(API_DOMAIN)
def _operate(self, action, search_limit):
index = 0
search_result = []
while True:
if search_limit != None and index > search_limit:
break
self._h.request("GET", action)
index = index + 25
res = loads(self._h.getresponse().read())
if res == []:
break
else: search_result += res
return search_result
def browse(self, sort_order=SORT_BY_DATES, index = 0):
"""
Browse Commandlinefu history, sorted by dates or by votes.
By default, CLF has 25 results per page, so index can be used
to see others 25 results.
"""
self._h.request("GET","/commands/browse/%s/json/%d"%(sort_order,index))
return self._h.getresponse().read()
def matching(self, match, sort_order = SORT_BY_DATES, search_limit = None):
"""
Search from history commands that match the "match" string.
By default, it returns all result, sort by dates with no search limit.
"""
match64 = encodestring(match)[:-1]
action = "/commands/matching/%s/%s/%s/json/%d"%(match, match64, sort_order, index)
return self._operate(action, search_limit)
def using(self, match, sort_order = SORT_BY_DATES, search_limit = None):
"""
Returns commands that implements the search word "match'.
By default, it returns all result, sort by dates with no search limit.
"""
action = "/commands/using/%s/%s/json/%d"%(match, sort_order, index)
return self._operate(action, search_limit)
def by(self, match, sort_order = SORT_BY_DATES, search_limit = None):
"""
Returns commands from a specified user.
By default, it returns all result, sort by dates with no search limit.
"""
action = "/commands/by/%s/%s/json/%d"%(match, sort_order, index)
return self._operate(action, search_limit)
class AmbariClient(object):
"""Quick Ambari wrapper
"""
def __init__(self, host, port, user, password):
self.conn = HTTPConnection(host=host, port=port)
self.get_headers = {
'Authorization' : 'Basic %s' % b64encode('%s:%s' % (user, password)),
'Content-Type': 'application/json',
'X-Requested-By': 'ambari'
}
self.write_headers = {
'Authorization' : 'Basic %s' % b64encode('%s:%s' % (user, password)),
'Content-Type': 'application/x-www-form-urlencoded',
'X-Requested-By': 'ambari'
}
def get(self, endpoint):
"""GET with JSON result
"""
self.conn.request('GET', endpoint, headers=self.get_headers)
response = self.conn.getresponse()
if response.status not in xrange(200, 207):
logger.debug('GET %s' % endpoint)
logger.debug(response.read())
raise RuntimeError('Error connecting to %s, status=%s' % (endpoint, response.status))
content = response.read()
if content:
return json.loads(content)
else:
return None
def _write(self, method, endpoint, data):
self.conn.request(method, endpoint, headers=self.write_headers, body=data)
response = self.conn.getresponse()
if response.status not in xrange(200, 207):
logger.debug('POST %s' % endpoint)
logger.debug(response.read())
raise RuntimeError('Error connecting to %s, status=%s' % (endpoint, response.status))
content = response.read()
if content:
return json.loads(content)
else:
return None
def put(self, endpoint, data):
"""PUT request
"""
self._write('PUT', endpoint, data)
def post(self, endpoint, data):
"""POST request
"""
self._write('POST', endpoint, data)
def append_print_file(print_id, file_path, file_contents, apibase, password):
""" Upload a file via the API append.php form input provision thingie.
"""
s, host, path, p, q, f = urlparse(apibase)
host, port = (':' in host) and host.split(':') or (host, 80)
query = urlencode({'print': print_id, 'password': password,
'dirname': dirname(file_path),
'mimetype': (guess_type(file_path)[0] or '')})
req = HTTPConnection(host, port)
req.request('GET', path + '/append.php?' + query)
res = req.getresponse()
html = ElementTree.parse(res)
for form in html.findall('*/form'):
form_action = form.attrib['action']
inputs = form.findall('.//input')
file_inputs = [input for input in inputs if input.attrib['type'] == 'file']
fields = [(input.attrib['name'], input.attrib['value'])
for input in inputs
if input.attrib['type'] != 'file' and 'name' in input.attrib]
files = [(input.attrib['name'], basename(file_path), file_contents)
for input in inputs
if input.attrib['type'] == 'file']
if len(files) == 1:
base_url = [el.text for el in form.findall(".//*") if el.get('id', '') == 'base-url'][0]
resource_url = urljoin(base_url, file_path)
post_type, post_body = encode_multipart_formdata(fields, files)
s, host, path, p, query, f = urlparse(urljoin(apibase, form_action))
host, port = (':' in host) and host.split(':') or (host, 80)
req = HTTPConnection(host, port)
req.request('POST', path+'?'+query, post_body, {'Content-Type': post_type, 'Content-Length': str(len(post_body))})
res = req.getresponse()
# res.read().startswith("Sorry, encountered error #1 ")
assert res.status in range(200, 308), 'POST of file to %s resulting in status %s instead of 2XX/3XX' % (host, res.status)
return resource_url
raise Exception('Did not find a form with a file input, why is that?')
def setUp(self):
from httplib import HTTPResponse
with Stub() as HTTPResponse:
HTTPResponse.status >> 200
HTTPResponse.read() >> '{"id": "THE-PRECIOUS-GUID", "title": "example title", "subtitle": "example title - subtitle", "description": "a description", "user": "******", "tags": [], "created_at": 1329767353.0, "source": "http://www.example.com/example.csv", "link": "http://www.junar.com/datastreams/some-url"}'
with Stub() as conn:
from httplib import HTTPConnection
conn = HTTPConnection("api.junar.com")
conn.request("POST", "/datastreams/publish", self.params, self.headers)
conn.getresponse() >> HTTPResponse
def stopZope(port, auth):
params = urlencode({'manage_shutdown:action': 'Shutdown'})
headers = {
'Content-Type': 'Application/x-www-form-urlencoded',
'Accept': 'text/plain',
'Authorization': 'Basic %s' % encodestring(auth)
}
conn = HTTPConnection('127.0.0.1:8080')
conn.request('POST', '/Control_Panel', params, headers)
# Throw the response away
conn.getresponse().read()
conn.close()
class ProductTest(TestCase):
def setUp(self):
self.conn = HTTPConnection('localhost', 5000, timeout=10)
self.conn.set_debuglevel(DEBUG_LEVEL)
def test_product_should_answer_200(self):
self.conn.request(GET, '/product')
response = self.conn.getresponse()
self.assertEquals(response.status, 200)
def test_product_id_should_answer_200(self):
self.conn.request(GET, '/product/0')
response = self.conn.getresponse()
self.assertEquals(response.status, 200)
def _make_request(method, host, path, body='', headers={}):
"""
Makes an HTTP request.
Args:
method |str| = Type of request. GET, POST, PUT or DELETE.
host |str| = The host and port of the server. E.g. vmpooler.myhost.com:8080
path |str| = The path of the url. E.g. /vm/vm_name
body |str| = The body data to send with the request.
headers |{str:str}| = Optional headers for the request.
Returns:
|HTTPResponse| = Response from the request.
Raises:
|RuntimeError| = If the vmpooler URL can't be reached
"""
try:
conn = HTTPConnection(host)
conn.request(method, path, body, headers)
resp = conn.getresponse()
return resp
except gaierror:
error = "Couldn't connect to address '{}'. Ensure this is the correct URL for " \
"the vmpooler".format(host)
raise RuntimeError(error)
except:
print("Unkown error occured while trying to connect to {}".format(host))
raise
def _publish(args, server, uri):
from httplib import HTTPConnection
from urllib import urlencode
args = dict((k,v) for k,v in args.items() if v != 'NA')
uri = uri + "?" + urlencode(args)
log.debug('Connect to: http://%s' % server)
log.debug('GET %s' % uri)
conn = HTTPConnection(server)
if not conn:
raise PublishException('Remote server connection error')
conn.request("GET", uri)
try:
http = conn.getresponse()
data = (http.status, http.reason, http.read())
conn.close()
if not (data[0] == 200 and data[1] == 'OK'):
raise PublishException('Server returned invalid status: %d %s %s'
% data)
except (DownloadError):
logging.warning("Download error: ", sys.exc_info()[0])
data = None
return data
def testGet(self):
self.assertTrue(self.pasteThread.isAlive())
from httplib import HTTPConnection
http_connection = HTTPConnection("localhost", port=self.port)
http_connection.request("GET", "/api/Crawl")
response = http_connection.getresponse()
self.assertTrue(response.status == 404 or response.status == 200 or response.status == 500)
def request(self, method, endpoint, qargs={}, data=None):
path = self.format_path(endpoint, qargs)
conn = HTTPConnection('%s:80' % self.account)
auth = b64encode("%s:%s" % (self.api_key, self.password))
headers = {
'Authorization': 'Basic %s' % auth,
'Content-Type': 'application/xml'
}
done = False
while not done:
try:
conn.request(method, path, headers=headers, body=data)
except socket.gaierror:
if self.retry_on_socket_error:
time.sleep(self.retry_timeout)
continue
else:
raise
resp = conn.getresponse()
body = resp.read()
if resp.status == 503 and self.retry_on_503:
time.sleep(self.retry_timeout)
else:
done = True
if 200 <= resp.status < 300:
return body
else:
raise ApiError("%s request to %s returned: %s\n%s" %
(method, path, resp.status, body), resp.status)
def search_on_playdeb(orig_file):
global MIRROR_URL, GETDEB_SUBDIR
http_connection = HTTPConnection(MIRROR_URL)
basename = orig_file.split('_')[0]
download_dir = 'http://' + MIRROR_URL + '/' + GETDEB_SUBDIR + '/ubuntu/pool/games/' + get_package_subdir(orig_file) + \
'/' + basename + '/'
http_connection.request('GET', download_dir)
http_response = http_connection.getresponse()
if http_response.status != 200: return None
data = http_response.read()
http_connection.close()
data = data.split('\n')
package_lines = list()
for line in data:
if basename in line:
package_lines.append(line)
if len(package_lines) == 0: return None
p_d = list()
package_re = re.compile('<a .*?>(?P<orig>.*?)(?:\.diff\.gz|\.debian\.tar\.gz)<')
download_re = re.compile('<a href="(?P<download>.*?)">')
for line in package_lines:
search_result = re.search(package_re, line)
if not search_result: continue
orig = search_result.group('orig')
search_result = re.search(download_re, line)
download = download_dir + search_result.group('download')
p_d.append((orig,download))
return p_d
def send_email(request):
try:
recipients = request.GET['to'].split(',')
url = request.GET['url']
proto, server, path, query, frag = urlsplit(url)
if query: path += '?' + query
conn = HTTPConnection(server)
conn.request('GET',path)
resp = conn.getresponse()
assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason)
rawData = resp.read()
conn.close()
message = MIMEMultipart()
message['Subject'] = "Graphite Image"
message['To'] = ', '.join(recipients)
message['From'] = 'composer@%s' % gethostname()
text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(),url) )
image = MIMEImage( rawData )
image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png"))
message.attach(text)
message.attach(image)
s = SMTP(settings.SMTP_SERVER)
s.sendmail('composer@%s' % gethostname(),recipients,message.as_string())
s.quit()
return HttpResponse( "OK" )
except:
return HttpResponse( format_exc() )
class HTTPSpeakerClient:
"""Emacspeak HTTP speech client, for HTTPSpeaker instances."""
def __init__(self, host="127.0.0.1", port=8000):
"Initialize client to connect to server at given host and port."
self._connection=HTTPConnection(host, port)
def postCommand(self, command, arg=""):
"""Post command, with argument arg (default, empty), to the speech server.
Returns the body of the server's HTTP response, if any. On error, HTTPSpeakerError is raised."""
body = command
if arg:
body += ": " + arg
self._connection.request("POST", "/", body, {"Content-type": "text/plain"})
response=self._connection.getresponse()
if response.status != 200:
raise HTTPSpeakerError(response.status, response.reason)
return response.read()
def speak(self, text):
"Speak the supplied string."
self.postCommand("speak", text)
def stop(self):
"Stop speaking."
self.postCommand("stop")
def isSpeaking(self):
"Return '0' when not speaking."
return self.postCommand("isSpeaking")
def close(self):
"Close the connection to the speech server."
self._connection.close()
class RESTClient(object):
"""
HTTP client
HTTPS client based on the provided URL (http:// or https://)
"""
def __init__(self, url, cert=None, key=None):
logging.info("RESTClient URL: %s" % url)
if url.startswith("https://"):
logging.info("Using HTTPS protocol, getting user identity files ...")
proxy_file = "/tmp/x509up_u%s" % os.getuid()
if not os.path.exists(proxy_file):
proxy_file = "UNDEFINED"
cert_file = cert or os.getenv("X509_USER_CERT", os.getenv("X509_USER_PROXY", proxy_file))
key_file = key or os.getenv("X509_USER_KEY", os.getenv("X509_USER_PROXY", proxy_file))
logging.info("Identity files:\n\tcert file: '%s'\n\tkey file: '%s' " % (cert_file, key_file))
url = url.replace("https://", "")
logging.info("Creating connection HTTPS ...")
self.conn = HTTPSConnection(url, key_file=key_file, cert_file=cert_file)
if url.startswith("http://"):
logging.info("Using HTTP protocol, creating HTTP connection ...")
url = url.replace("http://", "")
self.conn = HTTPConnection(url)
def http_request(self, verb, uri, data=None, header=None):
logging.debug("Request: %s %s %s ..." % (verb, uri, data))
self.conn.request(verb, uri, body=data, headers=self.headers)
resp = self.conn.getresponse()
data = resp.read()
logging.debug("Status: %s" % resp.status)
logging.debug("Reason: %s" % resp.reason)
return resp.status, data
def urlopen(url, svprev, formdata):
ua = "SPlayer Build %d" % svprev
#prepare data
#generate a random boundary
boundary = "----------------------------" + "%x"%random.getrandbits(48)
data = []
for item in formdata:
data.append("--" + boundary + "\r\nContent-Disposition: form-data; name=\"" + item[0] + "\"\r\n\r\n" + item[1] + "\r\n")
data.append("--" + boundary + "--\r\n")
data = "".join(data)
cl = str(len(data))
r = urlparse(url)
h = HTTPConnection(r.hostname)
h.connect()
h.putrequest("POST", r.path, skip_host=True, skip_accept_encoding=True)
h.putheader("User-Agent", ua)
h.putheader("Host", r.hostname)
h.putheader("Accept", "*/*")
h.putheader("Content-Length", cl)
h.putheader("Expect", "100-continue")
h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary)
h.endheaders()
h.send(data)
resp = h.getresponse()
if resp.status != OK:
raise Exception("HTTP response " + str(resp.status) + ": " + resp.reason)
return resp
def check_url_path(path, redirected=0):
if redirected > MAX_REDIRECTION_ALLOWED:
return False
try:
code = ''
parse_result = urlparse(path)
server_name = parse_result.netloc
urlpath = parse_result.path
if not urlpath:
# Just a server, as with a repo.
with contextlib.closing(urllib2.urlopen(path)) as res:
code = res.getcode()
else:
# socket.gaierror could be raised,
# which is a child class of IOError
conn = HTTPConnection(server_name, timeout=15)
# Don't try to get the whole file:
conn.request('HEAD', path)
response = conn.getresponse()
code = response.status
conn.close()
if code == 200:
return True
elif code == 301 or code == 302:
for header in response.getheaders():
if header[0] == 'location':
return check_url_path(header[1], redirected+1)
else:
return False
except (urllib2.URLError, HTTPException, IOError, ValueError):
return False
return True
def getresponse(self):
r = LibcloudHTTPConnection.getresponse(self)
if self.log is not None:
r, rv = self._log_response(r)
self.log.write(rv + "\n")
self.log.flush()
return r
def proxy(request):
if 'url' not in request.GET:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain"
)
url = urlsplit(request.GET['url'])
# Don't allow localhost connections unless in DEBUG mode
if not settings.DEBUG and re.search('localhost|127.0.0.1', url.hostname):
return HttpResponse(status=403)
locator = url.path
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
# Strip all headers and cookie info
headers = {}
conn = HTTPConnection(url.hostname, url.port) if url.scheme == "http" else HTTPSConnection(url.hostname, url.port)
conn.request(request.method, locator, request.raw_post_data, headers)
result = conn.getresponse()
response = HttpResponse(
valid_response(result.read()),
status=result.status,
content_type=result.getheader("Content-Type", "text/plain")
)
return response
def _select(self, params):
# encode the query as utf-8 so urlencode can handle it
params['q'] = unicode_safe(params['q'])
path = '%s/select/?%s' % (self.path, urlencode(params))
conn = HTTPConnection(self.host, self.port)
conn.request('GET', path)
return conn.getresponse()
def proxy(request):
if 'url' not in request.GET:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain")
url = urlsplit(request.GET['url'])
locator = url.path
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
headers = {}
if settings.SESSION_COOKIE_NAME in request.COOKIES:
headers["Cookie"] = request.META["HTTP_COOKIE"]
if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
headers["Content-Type"] = request.META["CONTENT_TYPE"]
if request.META.get('HTTP_AUTHORIZATION'):
headers['AUTHORIZATION'] = request.META.get('HTTP_AUTHORIZATION')
conn = HTTPConnection(url.hostname, url.port)
conn.request(request.method, locator, request.raw_post_data, headers)
result = conn.getresponse()
response = HttpResponse(result.read(),
status=result.status,
content_type=result.getheader(
"Content-Type", "text/plain"))
if result.getheader('www-authenticate'):
response['www-authenticate'] = result.getheader('www-authenticate')
return response
def download_api_data(filename, coord, api_base, projection):
""" Download API data for a tile to a named file, return size in kilobytes.
"""
s, host, path, p, q, f = urlparse(api_base)
bbox = coordinate_latlon_bbox(coord, projection)
path = join(path, 'api/0.6/map?bbox=%.6f,%.6f,%.6f,%.6f' % bbox)
conn = HTTPConnection(host)
conn.request('GET', path, headers={'Accept-Encoding': 'compress, gzip'})
resp = conn.getresponse()
assert resp.status == 200, (resp.status, resp.read())
if resp.getheader('Content-Encoding') == 'gzip':
disk = open(filename, 'w')
else:
raise Exception((host, path))
disk = GzipFile(filename, 'w')
bytes = resp.read()
disk.write(bytes)
disk.close()
return len(bytes) / 1024.
def contact_vessel(self, vessel_ip, path, action, key, value):
# the Boolean variable we will return
success = False
# The variables must be encoded in the URL format, through urllib.urlencode
post_content = urlencode({
'action': action,
'key': key,
'value': value
})
# the HTTP header must contain the type of data we are transmitting, here URL encoded
headers = {"Content-type": "application/x-www-form-urlencoded"}
# We should try to catch errors when contacting the vessel
try:
# We contact vessel:PORT_NUMBER since we all use the same port
# We can set a timeout, after which the connection fails if nothing happened
connection = HTTPConnection("%s:%d" % (vessel_ip, PORT_NUMBER),
timeout=30)
# We only use POST to send data (PUT and DELETE not supported)
action_type = "POST"
# We send the HTTP request
connection.request(action_type, path, post_content, headers)
# We retrieve the response
response = connection.getresponse()
# We want to check the status, the body should be empty
status = response.status
# If we receive a HTTP 200 - OK
if status == 200:
success = True
# We catch every possible exceptions
except Exception as e:
print "Error while contacting %s" % vessel_ip
# printing the error given by Python
print(e)
# we return if we succeeded or not
return success
def GetSesc(self):
print "[+] Trying to read Sesc"
for i in range(0, 2):
conn = HTTPConnection(target, port)
conn.request("GET", path + "index.php?action=pm;sa=manlabels;", {},
{
"Accept": "text/plain",
"Cookie": sn + "=" + sv + ";"
})
rsp = conn.getresponse()
r = rsp.read()
if rsp.status == 404:
exit("[-] Error 404. Not Found")
elif r.find('<input type="hidden" name="sc" value="') != -1 and r.find(
'" />') != -1:
self.sesc = r.split(
'<input type="hidden" name="sc" value="')[1].split('" />')[0]
if len(self.sesc) != 32: exit("[-] Invalid Sesc")
print "[+] Sesc has been successfully read ==> " + self.sesc
else:
exit("[-] Unable to find Sesc")
def notifyNodeServer(result):
ret = False
notifyPayload = dictToJson(result)
log.msg(
'[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer parms: '
% (game, tradeNo), notifyPayload)
try:
nodeHttpServer = urlparse.urlparse(
getAppConfig('node.httpserver',
orderServerOption.get('env')))
nodeHttpHost = nodeHttpServer.hostname
nodeHttpPort = nodeHttpServer.port
httpClient = HTTPConnection(host=nodeHttpHost,
port=nodeHttpPort)
endPoint = "/api/games/reCharge"
httpClient.request(
'POST',
endPoint,
notifyPayload,
headers={'Content-Type': 'application/json'})
response = httpClient.getresponse()
log.msg(
'[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer : '
% (game, tradeNo), response.status, response.reason)
log.msg(
'[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer : '
% (game, tradeNo), response.read())
httpClient.close()
ret = True
except Exception, e:
log.msg(
'[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer fail: '
% (game, tradeNo), e.messag)
log.msg(
'[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer trace: '
% (game, tradeNo), e)
def s3save(url, data):
auth = "LOW %s:%s" % (appauth.key, appauth.secret)
c = HTTPConnection("s3.us.archive.org")
c.request("PUT", url, data, {"Authorization": auth})
r = c.getresponse()
def fproxyGet(self, path):
"""
Fetches from fproxy, returns (status, mimetype, data)
"""
server = self.server
headers = self.headers
print "--------------------------------------------"
print "** path=%s" % path
# first scenario - user is pointing their browser directly at
# fproxyfproxy, barf!
if not path.startswith("http://"):
self.send_response(400)
self.send_header("Content-type", "text/html")
data = "\n".join([
"<html><head>",
"<title>Access Denied</title>",
"</head><body>",
"<h1>Access Denied</h1>",
"Sorry, but FProxyProxy is an http proxy server.<br>",
"Please don't try to access it like a web server.",
"</body></html>",
"",
])
self.send_header("Content-Length", str(len(data)))
self.send_header("Location", location)
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
return
# convert path to relative
path = "/" + path[7:].split("/", 1)[-1]
#print "** path=%s" % repr(path)
try:
# check host header
hostname = headers.get("Host", 'fproxy')
pathbits = path.split("/")
print "** hostname = %s" % hostname
# second scenario, user has just given a domain name without trailing /
if len(pathbits) == 1:
# redirect to force trailing slash
location = path + "/"
print "** redirecting to: %s" % location
self.send_response(301)
self.send_header("Content-type", "text/html")
data = "\n".join([
"<html><head>",
"<title>Permanent redirect: new URI</title>",
"</head><body>",
"<h1>Permanent redirect: new URI</h1>",
"<a href=\"%s\">Click here</a>",
"</body></html>",
"",
]) % location
self.send_header("Content-Length", str(len(data)))
self.send_header("Location", location)
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
return
tail = "/".join(pathbits[1:])
# third scenario - request into fproxy
if hostname == 'fproxy':
# tis an fproxy request, go straight through
conn = HTTPConnection(server.fproxyHost, server.fproxyPort)
conn.request("GET", path)
resp = conn.getresponse()
self.send_response(resp.status)
self.send_header("Content-type",
resp.getheader("Content-Type", "text/plain"))
data = resp.read()
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
conn.close()
return
else:
# final scenario - some other domain, try lookup
uri = server.node.namesiteLookup(hostname)
if not uri:
# lookup failed, do the usual 404 thang
print "** lookup of domain %s failed" % hostname
self.send_response(404)
self.send_header("Content-type", "text/html")
data = "\n".join([
"<html><head>",
"<title>404 - Freenet name not found</title>",
"</head><body",
"<h1>404 - Freenet name not found</title>",
"The pyFreenet name service was unable to resolve ",
"the name %s" % hostname,
"<br><br>",
"You might like to find its freenet uri and try that ",
"within <a href=\"/fproxy/\">FProxy</a>",
"</body></html>",
"",
])
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
# lookup succeeded - ok to go now via fproxy
conn = HTTPConnection(server.fproxyHost, server.fproxyPort)
newpath = "/" + uri
if tail:
if not newpath.endswith("/"):
newpath += "/"
newpath += tail
print "** newpath=%s" % newpath
conn.request("GET", newpath)
resp = conn.getresponse()
print "** status=%s" % resp.status
self.send_response(resp.status)
self.send_header("Content-type",
resp.getheader("Content-Type", "text/plain"))
# has fproxy sent us a redirect?
if resp.status == 301:
# yuck, fproxy is telling us to redirect, which
# sadly means we have to lose the domain name
# from our browser address bar
location = resp.getheader("location")
newLocation = "http://fproxy" + location
print "*** redirected!!!"
print "*** old location = %s" % location
print "*** --> %s" % newLocation
self.send_header("Location", newLocation)
# get the data from fproxy and send it up to the client
data = resp.read()
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
self.wfile.flush()
conn.close()
return
return
except socket.error:
raise
def _head(cls, reference):
conn = HTTPConnection(str(reference.authority))
# XXX Add the query
conn.request('HEAD', str(reference.path))
return conn.getresponse()
class GoogleSuggestions():
def __init__(self):
self.hl = "en"
self.conn = None
def prepareQuery(self):
#GET /complete/search?output=toolbar&client=youtube-psuggest&xml=true&ds=yt&hl=en&jsonp=self.gotSuggestions&q=s
#self.prepQuerry = "/complete/search?output=toolbar&client=youtube&xml=true&ds=yt&"
self.prepQuerry = "/complete/search?output=chrome&client=chrome&"
if self.hl is not None:
self.prepQuerry = self.prepQuerry + "hl=" + self.hl + "&"
self.prepQuerry = self.prepQuerry + "jsonp=self.gotSuggestions&q="
print("[MyTube - GoogleSuggestions] prepareQuery:", self.prepQuerry)
def getSuggestions(self, queryString):
self.prepareQuery()
if queryString is not "":
query = self.prepQuerry + quote(queryString)
self.conn = HTTPConnection("google.com")
try:
self.conn = HTTPConnection("google.com")
self.conn.request("GET", query, "",
{"Accept-Encoding": "UTF-8"})
except (CannotSendRequest, gaierror, error):
self.conn.close()
print(
"[MyTube - GoogleSuggestions] Can not send request for suggestions"
)
return None
else:
try:
response = self.conn.getresponse()
except BadStatusLine:
self.conn.close()
print(
"[MyTube - GoogleSuggestions] Can not get a response from google"
)
return None
else:
if response.status == 200:
data = response.read()
header = response.getheader(
"Content-Type", "text/xml; charset=ISO-8859-1")
charset = "ISO-8859-1"
try:
charset = header.split(";")[1].split("=")[1]
print(
"[MyTube - GoogleSuggestions] Got charset %s" %
charset)
except:
print(
"[MyTube - GoogleSuggestions] No charset in Header, falling back to %s"
% charset)
data = data.decode(charset).encode("utf-8")
self.conn.close()
return data
else:
self.conn.close()
return None
else:
return None
class TestRegression(TestCase):
def setUp(self):
self.ip = "localhost"
self.port = "8081"
self.headers = {'Content-Type': 'application/json',
'Accept': 'application/json'}
self.timeout = 10
# connect
self.conn = HTTPConnection(self.ip, self.port, timeout=self.timeout)
def tearDown(self):
self.conn.close()
def test_1_basic_request(self):
"""
"""
# request an pa computation
path = "/5gt/so/v1/PAComp"
data = {"ReqId": "string",
"nfvi": {
"NFVIPoPs": [
{
"Id": "DC1",
"gw_ip_address": "172.16.102.116",
"capabilities": {
"cpu": 3,
"ram": 2400,
"storage": 135000
},
"availableCapabilities": {
"cpu": 78,
"ram": 2400,
"storage": 135000
},
"internal_latency": 1.04
},
{
"Id": "DC2",
"gw_ip_address": "172.16.102.117",
"capabilities": {
"cpu": 3,
"ram": 160,
"storage": 7000
},
"availableCapabilities": {
"cpu": 85,
"ram": 160,
"storage": 7000
},
"internal_latency": 4.16
},
{
"Id": "DC3",
"gw_ip_address": "172.16.102.118",
"capabilities": {
"cpu": 2,
"ram": 160,
"storage": 7000
},
"availableCapabilities": {
"cpu": 40,
"ram": 160,
"storage": 7000
},
"internal_latency": 4.16
}
],
"LLs": [
{
"LLid": "LL1",
"capacity": {
"total": 125000000000,
"available": 1250000000
},
"delay": 3.3,
"length": 250,
"source": {
"Id": "DC1",
"GwIpAddress": "172.16.102.116"
},
"destination": {
"Id": "DC2",
"GwIpAddress": "172.16.102.117"
}
},
{
"LLid": "LL2",
"capacity": {
"total": 1250000000000,
"available": 1250000000
},
"delay": 1.3,
"length": 150,
"source": {
"Id": "DC1",
"GwIpAddress": "172.16.102.116"
},
"destination": {
"Id": "DC3",
"GwIpAddress": "172.16.102.118"
}
}
]
},
"nsd": {
"Id": "Req1",
"name": "req1",
"VNFs": [
{
"VNFid": "VNFa",
"instances": 1,
"requirements": {
"cpu": 3,
"ram": 8,
"storage": 1
}
},
{
"VNFid": "VNFb",
"instances": 2,
"requirements": {
"cpu": 4,
"ram": 5,
"storage": 2
}
}
],
"VNFLinks": [
{
"source": "172.16.102.116",
"required_capacity": 1250000
}
],
"max_latency": 9.6
},
"callback": "string"
}
self.conn.request("POST", path, dumps(data), self.headers)
self.conn.sock.settimeout(self.timeout)
rsp = self.conn.getresponse()
data = rsp.read()
self.assertEqual(rsp.status, 201)
def run(self):
headers = {
'Ntrip-Version':
'Ntrip/2.0',
'User-Agent':
'NTRIP ntrip_ros',
'Connection':
'close',
'Authorization':
'Basic ' +
b64encode(self.ntc.ntrip_user + ':' + str(self.ntc.ntrip_pass))
}
connection = HTTPConnection(self.ntc.ntrip_server)
connection.request('GET', '/' + self.ntc.ntrip_stream,
self.ntc.nmea_gga, headers)
response = connection.getresponse()
if response.status != 200:
raise Exception("Error: Response not HTTP200 (OK)")
buf = ""
rmsg = Message()
restart_count = 0
while not self.stop:
'''
data = response.read(100)
pos = data.find('\r\n')
if pos != -1:
rmsg.message = buf + data[:pos]
rmsg.header.seq += 1
rmsg.header.stamp = rospy.get_rostime()
buf = data[pos+2:]
self.ntc.pub.publish(rmsg)
else: buf += data
'''
''' This now separates individual RTCM messages and publishes each one on the same topic '''
data = response.read(1)
if len(data) != 0:
if ord(data[0]) == 211:
buf += data
data = response.read(2)
buf += data
cnt = ord(data[0]) * 256 + ord(data[1])
data = response.read(2)
buf += data
typ = (ord(data[0]) * 256 + ord(data[1])) / 16
print(str(datetime.now()), cnt, typ)
cnt = cnt + 1
for x in range(cnt):
data = response.read(1)
buf += data
rmsg.message = buf
rmsg.header.seq += 1
rmsg.header.stamp = rospy.get_rostime()
self.ntc.pub.publish(rmsg)
buf = ""
else:
print(data)
else:
''' If zero length data, close connection and reopen it '''
restart_count = restart_count + 1
print("Zero length ", restart_count)
connection.close()
connection = HTTPConnection(self.ntc.ntrip_server)
connection.request('GET', '/' + self.ntc.ntrip_stream,
self.ntc.nmea_gga, headers)
response = connection.getresponse()
if response.status != 200:
raise Exception("Error: Response not HTTP200 (OK)")
buf = ""
connection.close()
class HTTPRelayClient:
def __init__(self, target):
# Target comes as protocol://target:port/path
self.target = target
proto, host, path = target.split(':')
host = host[2:]
self.path = '/' + path.split('/', 1)[1]
if proto.lower() == 'https':
#Create unverified (insecure) context
try:
uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
self.session = HTTPSConnection(host, context=uv_context)
except AttributeError:
#This does not exist on python < 2.7.11
self.session = HTTPSConnection(host)
else:
self.session = HTTPConnection(host)
self.lastresult = None
def sendNegotiate(self, negotiateMessage):
#Check if server wants auth
self.session.request('GET', self.path)
res = self.session.getresponse()
res.read()
if res.status != 401:
logging.info(
'Status code returned: %d. Authentication does not seem required for URL'
% res.status)
try:
if 'NTLM' not in res.getheader('WWW-Authenticate'):
logging.error(
'NTLM Auth not offered by URL, offered protocols: %s' %
res.getheader('WWW-Authenticate'))
return False
except (KeyError, TypeError):
logging.error(
'No authentication requested by the server for url %s' %
self.target)
return False
#Negotiate auth
negotiate = base64.b64encode(negotiateMessage)
headers = {'Authorization': 'NTLM %s' % negotiate}
self.session.request('GET', self.path, headers=headers)
res = self.session.getresponse()
res.read()
try:
serverChallengeBase64 = re.search(
'NTLM ([a-zA-Z0-9+/]+={0,2})',
res.getheader('WWW-Authenticate')).group(1)
serverChallenge = base64.b64decode(serverChallengeBase64)
return serverChallenge
except (IndexError, KeyError, AttributeError):
logging.error('No NTLM challenge returned from server')
def sendAuth(self, authenticateMessageBlob, serverChallenge=None):
#Negotiate auth
auth = base64.b64encode(authenticateMessageBlob)
headers = {'Authorization': 'NTLM %s' % auth}
self.session.request('GET', self.path, headers=headers)
res = self.session.getresponse()
if res.status == 401:
return False
else:
logging.info(
'HTTP server returned error code %d, treating as a succesful login'
% res.status)
#Cache this
self.lastresult = res.read()
return True
#SMB Relay server needs this
@staticmethod
def get_encryption_key():
return None
def test_redirects(self):
''' Check a selection of HTTP redirect pairs.
'''
pairs = [
('/accelerator', '/companies/accelerator-faq/'),
('/incubator', '/companies/incubator-faq/'),
('/projects', '/apps/'),
('/brigade/projects', '/brigade/projects'),
('/focus', '/our-work/focus-areas/'),
('/governments/capabilities', '/governments/principles/'),
('/governments/capabilities/index.html',
'/governments/principles/index.html'),
('/governments/capabilities/open-data',
'/governments/principles/open-data/'),
('/procurement', '/governments/principles/procurement/'),
('/governments/capabilities/procurement',
'/governments/principles/procurement/'),
('/team', '/about/team/'),
('/projects', '/apps/'),
('/fellows/job-description', '/geeks/fellowship-faq/'),
('/fellows/apply', '/geeks/fellowship-apply/'),
('/fellows/alumni-fellows', '/geeks/our-geeks/alumni-fellows/'),
('/alumni-fellows', '/geeks/our-geeks/alumni-fellows/'),
('/fellows/2011-fellows', '/geeks/our-geeks/2011-fellows/'),
('/2011-fellows', '/geeks/our-geeks/2011-fellows/'),
('/fellows/2012-fellows', '/geeks/our-geeks/2012-fellows/'),
('/2012-fellows', '/geeks/our-geeks/2012-fellows/'),
('/fellows/2013-fellows', '/geeks/our-geeks/2013-fellows/'),
('/2013-fellows', '/geeks/our-geeks/2013-fellows/'),
('/fellows/2014-fellows', '/geeks/our-geeks/2014-fellows/'),
('/2014-fellows', '/geeks/our-geeks/2014-fellows/'),
('/startups', '/about/companies/'),
('/fellowship', '/about/fellowship/'),
# ('/fellows', '/about/fellowship/'),
('/procurement', '/governments/principles/procurement/'),
('/austin', '/governments/austin/'),
('/honolulu', '/governments/honolulu/'),
('/alex-pandel', '/people/alex-pandel/'),
('/alex-yule', '/people/alex-yule/'),
('/cities', '/governments/'),
('/cities/2015-partners', '/governments/2015-partners/'),
('/codeacross-2014', '/events/codeacross-2014/'),
('/codeacross', '/events/codeacross-2015/'),
('/02-18-2014', '/peer-network-training/02-18-2014/'),
('/brigade12-12-2013', '/brigade-training/brigade12-12-2013/'),
('/09-19-2013', '/brigade-training/09-19-2013/'),
('/blog/meettheauthors', '/peer-network-training/meettheauthors/'),
('/09-24-2013', '/peer-network-training/09-24-2013/'),
('/blog/08-22-2013', '/peer-network-training/08-22-2013/'),
('/08-07-2013', '/peer-network-training/08-07-2013/'),
('/blog/ask-a-fellow-2013',
'/peer-network-training/ask-a-fellow-2013/'),
('/accelerator', '/companies/accelerator-faq/'),
('/accelerator-2013', '/companies/accelerator-faq/'),
('/accelerator-2', '/companies/accelerator-faq/'),
('/startups/accelerator', '/companies/accelerator-faq/'),
('/incubator', '/companies/incubator-faq/'),
('/incubator-2', '/companies/incubator-faq/'),
('/startups/incubator-2', '/companies/incubator-faq/'),
('/who-we-are', '/about/team/'),
('/donors', '/supporters/'),
('/how-to-help', '/geeks/'),
('/code-across-america', '/events/codeacross-2012/'),
('/fellows/faq', '/geeks/fellowship-faq/'),
('/fellows/training', '/geeks/fellowship-faq/'),
('/fellows/institute', '/geeks/fellowship-faq/'),
('/governments/expectations-of-our-city-partners',
'/governments/fellowship/'),
('/2012-city-videos', '/communications/testimonial/'),
('/geeks/fellows/alumni-fellows',
'/geeks/our-geeks/alumni-fellows/'),
('/fellows/current-fellows', '/geeks/our-geeks/2013-fellows/'),
('/fellows/alumni-fellows/2013-fellows',
'/geeks/our-geeks/2013-fellows/'),
('/geeks/our-startups', '/companies/our-companies/'),
('/geeks/accelerator-faq', '/companies/accelerator-faq/'),
('/geeks/accelerator-apply', '/companies/accelerator-apply/'),
('/geeks/fellowship', '/about/fellowship/'),
('/governments/city-impact', '/about/fellowship/'),
('/the-program', '/about/fellowship/'),
# ('/about/fellowship', '/governments/fellowship/'),
('/governments/fellowship-faq', '/governments/fellowship/'),
('/governments/2013-apply-now', '/governments/fellowship/'),
('/governments/apply', '/forms/governments/fellowship/apply/'),
('/governments/apply-now', '/forms/governments/fellowship/apply/'),
('/governments/fellowship-apply',
'/forms/governments/fellowship/apply/'),
('/forms/governments/fellowship/interest',
'/forms/governments/fellowship/apply/'),
('/city-alumni', '/governments/alumni/'),
('/cfa-2012', '/governments/alumni/'),
('/2012', '/governments/alumni/'),
('/governments/faqs-cities-applying-to-program',
'/governments/fellowship/'),
('/civic-data-standards', '/governments/data-standards-faq/'),
('/application-info', '/governments/fellowship/'),
('/application-questions', '/governments/fellowship/'),
('/governments/call-for-applications', '/governments/fellowship/'),
('/2013-partners', '/governments/alumni/'),
('/city-current', '/governments/alumni/'),
('/governments/current', '/governments/alumni/'),
('/2013-partners/kansas-city', '/governments/kansascity/'),
('/2013-partners/louisville', '/governments/louisville/'),
('/2013-partners/las-vegas', '/governments/lasvegas/'),
('/city-current/kansas-city', '/governments/kansascity/'),
('/city-current/louisville', '/governments/louisville/'),
('/city-current/las-vegas', '/governments/lasvegas/'),
('/2012-partners/austin', '/governments/austin/'),
('/2012-partners/chicago', '/governments/chicago/'),
('/2012-partners/detroit', '/governments/detroit/'),
# ('/2012-city-finalists', '/2012-city-finalists/'),
('/2012-city-finalists/chicago', '/governments/chicago/'),
('/2012-city-finalists/detroit', '/governments/detroit/'),
('/city-alumni/boston', '/governments/boston/'),
('/city-alumni/chicago', '/governments/chicago/'),
('/city-alumni/philadelphia', '/governments/philadelphia/'),
('/governments/summit-county', '/governments/summitcounty/'),
('/focus', '/our-work/focus-areas/'),
('/projects/new-york-city-hhc-accelerator/',
'/projects/new-york-city-hhs-accelerator/'),
# # This actually goes to BSD, but we check for it anyway.
# ('/donate', '/page/contribute/default'),
]
for (start_path, end_path) in pairs:
url = 'http://0.0.0.0:{0}{1}'.format(self.port, start_path)
for i in range(100):
if i > 10:
raise Exception(
'Too many redirects from {0}, now at {1}'.format(
start_path, url_path))
_, url_host, url_path, _, _, _ = urlparse(url)
conn = HTTPConnection(*url_host.split(':'))
conn.request('GET', url_path)
resp = conn.getresponse()
conn.close()
if resp.status not in range(300, 399):
break
url = urljoin(url, resp.getheader('location'))
_, new_host, url_path, _, _, _ = urlparse(url)
# Stop if we go off-site
if new_host != url_host:
break
assert end_path == url_path, '{0} instead of {1} from {2}'.format(
url_path, end_path, start_path)
if slug not in files:
files[slug] = dict()
files[slug][key] = slug_file
coast['coastline-good'] = {}
for prj in ('merc', 'latlon'):
file = 'coastline-good-%s.tar.bz2' % prj
href = urljoin(base_url, file)
url = urlparse(href)
conn = HTTPConnection(url.netloc)
conn.request('HEAD', url.path)
resp = conn.getresponse()
size = resp.getheader('content-length')
date = parser.parse(resp.getheader('last-modified'))
date = '%s %d, %s' % (months[date.month], date.day, date.year)
coast['coastline-good'][prj] = (file, int(size), href, date)
cities = list(DictReader(open('cities.txt'), dialect='excel-tab'))
#
print >> index, """<!DOCTYPE html>
<html lang="en">
<head>
<title>Metro Extracts</title>
<meta http-equiv="content-type" content="text/html; charset=utf-8">
def makeRequest(url,
values=None,
verb='GET',
accept="text/plain",
contentType=None,
secure=False,
secureParam={}):
headers = {}
contentType = contentType or "application/x-www-form-urlencoded"
headers = {
"content-type": contentType,
"Accept": accept,
"cms-auth-status": "NONE"
}
if secure:
headers.update({
"cms-auth-status":
"OK",
"cms-authn-dn":
"/DC=ch/OU=Organic Units/OU=Users/CN=Fake User",
"cms-authn-name":
"Fake User",
"cms-authz-%s" % secureParam['role']:
"group:%s site:%s" % (secureParam['group'], secureParam['site'])
})
headers["cms-authn-hmac"] = _generateHash(secureParam["key"], headers)
data = None
if verb == 'GET' and values:
data = urllib.urlencode(values, doseq=True)
elif verb != 'GET' and values:
# needs to test other encoding type
if contentType == "application/x-www-form-urlencoded":
data = urllib.urlencode(values)
else:
# for other encoding scheme values assumed to be encoded already
data = values
parser = urlparse(url)
uri = parser.path
if parser.query:
uri += "?" + parser.query
if verb == 'GET' and data != None:
uri = '%s?%s' % (uri, data)
# need to specify Content-length for POST method
# TODO: this function needs refactoring - too verb-related branching
if verb != 'GET':
if data:
headers.update({"content-length": len(data)})
else:
headers.update({"content-length": 0})
conn = HTTPConnection(parser.netloc)
conn.connect()
conn.request(verb, uri, data, headers)
response = conn.getresponse()
data = response.read()
conn.close()
cType = response.getheader('content-type').split(';')[0]
return data, response.status, cType, response
def sendData(uri,
method="POST",
data_raw=None,
data_enc=None,
svar_fra_mottager=1,
tag='karusell',
reqursivLevel=1,
multipefaktor=1):
"""Sender gluoner til de forskjellige transmitterne.
Denne brukes av trådmodulen og blir kjørt en gang pr. transmitter"""
ret_svar = {}
utverdier = {}
#Lager midlertidig svar som ev. oversktives senere
#Lock og skriv
ret_svar['status'] = '0'
ret_svar['reason'] = 'Starter utsendelse'
utverdier[tag] = ret_svar
#start = time.time()
#Sett opp headers
if data_raw:
headers = {
"Content-type": "application/xml",
"Accept": "*/*",
"User-Agent": "Gluon 0.5.0 [no] (%s; U)" % sys.platform
}
data = data_raw
else:
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Accept": "*/*",
"User-Agent": "Gluon 0.5.0 [no] (%s; U)" % sys.platform
}
data = data_enc
#Dele opp uri til hostname og url
host, url = uri[7:].split('/', 1)
start = time.time()
try:
conn = HTTPConnection(host)
conn.request(method, '/' + url, data, headers)
ret_svar['status'] = '3'
ret_svar[
'reason'] = 'Sendt data - transmitter svarer ikke. Ting er sansynlighvis OK'
#Dette skyldes som regel en treg mottaker og er ikke nødvendighvis en feil
time.sleep(0.003)
except:
#Legge inn forskjellige verdier her
#Ev. legge inn rutine for automatisk forsøk igjen
ret_svar['status'] = '1'
ret_svar['reason'] = 'Kunne ikke lage forbindelse'
else:
if svar_fra_mottager:
svar = conn.getresponse()
#print dir(svar)
ret_svar['status'] = svar.status
ret_svar['reason'] = svar.reason
if method == 'HEAD':
ret_svar['msg'] = ''
else:
ret_svar['msg'] = svar.read()
ret_svar['duration'] = "PT%sS" % (time.time() - start)
else:
tid = time.time() - start
ret_svar['msg'] = 'TID: %s HVEM: %s' % (tid, tag)
conn.close()
#Sjekk om vi har en 302
if ret_svar['status'] == 302:
#Vi sender videre i denne omgang
if reqursivLevel < maxReqursivLevel:
nyUrl = svar.getheader('Location')
nytransmitter = tag + ":redirekt_%s" % reqursivLevel
sendData(nyUrl,
data_raw=data_raw,
svar_fra_mottager=svar_fra_mottager,
tag=nytransmitter,
reqursivLevel=reqursivLevel + 1)
#ret_svar['msg'] = nyUrl
#Lock og skriv
utverdier[tag] = ret_svar
return utverdier
try:
conf = open("./portconf", "r")
pport = conf.readline().rstrip().split(':')[1]
sport1 = conf.readline().rstrip().split(':')[1]
sport2 = conf.readline().rstrip().split(':')[1]
server1 = ServerThread(int(sport1))
server1.start()
dataFile = open('basic', "r")
cdata = dataFile.read()
r = False
proxy = '127.0.0.1:' + pport
conn = HTTPConnection(proxy)
conn.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
resp = conn.getresponse()
data = resp.read()
conn.close()
time.sleep(3)
conn2 = HTTPConnection(proxy)
conn2.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
resp2 = conn2.getresponse()
data2 = resp2.read()
conn2.close()
time.sleep(3)
conn3 = HTTPConnection(proxy)
conn3.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
resp3 = conn3.getresponse()
data3 = resp3.read()
class SingleDownload(SingleDownloadHelperInterface):
def __init__(self, downloader, url):
SingleDownloadHelperInterface.__init__(self)
self.downloader = downloader
self.baseurl = url
try:
self.scheme, self.netloc, path, pars, query, fragment = urlparse(url)
except:
self.downloader.errorfunc('cannot parse http seed address: ' + url)
return
if self.scheme != 'http':
self.downloader.errorfunc('http seed url not http: ' + url)
return
self.proxyhost = find_proxy(url)
try:
if self.proxyhost is None:
self.connection = HTTPConnection(self.netloc)
else:
self.connection = HTTPConnection(self.proxyhost)
except:
self.downloader.errorfunc('cannot connect to http seed: ' + url)
return
self.seedurl = path
if pars:
self.seedurl += ';' + pars
self.seedurl += '?'
if query:
self.seedurl += query + '&'
self.seedurl += 'info_hash=' + urllib.quote(self.downloader.infohash)
self.measure = Measure(downloader.max_rate_period)
self.index = None
self.url = ''
self.requests = []
self.request_size = 0
self.endflag = False
self.error = None
self.retry_period = 30
self._retry_period = None
self.errorcount = 0
self.goodseed = False
self.active = False
self.cancelled = False
self.resched(randint(2, 10))
def resched(self, len = None):
if len is None:
len = self.retry_period
if self.errorcount > 3:
len = len * (self.errorcount - 2)
self.downloader.rawserver.add_task(self.download, len)
def _want(self, index):
if self.endflag:
return self.downloader.storage.do_I_have_requests(index)
else:
return self.downloader.storage.is_unstarted(index)
def download(self):
if DEBUG:
print 'http-sdownload: download()'
if self.is_frozen_by_helper():
if DEBUG:
print 'http-sdownload: blocked, rescheduling'
self.resched(1)
return
self.cancelled = False
if self.downloader.picker.am_I_complete():
self.downloader.downloads.remove(self)
return
self.index = self.downloader.picker.next(haveall, self._want, self)
if self.index is None and self.frozen_by_helper:
self.resched(0.01)
return
if self.index is None and not self.endflag and not self.downloader.peerdownloader.has_downloaders():
self.endflag = True
self.index = self.downloader.picker.next(haveall, self._want, self)
if self.index is None:
self.endflag = True
self.resched()
else:
self.url = self.seedurl + '&piece=' + str(self.index)
self._get_requests()
if self.request_size < self.downloader.storage._piecelen(self.index):
self.url += '&ranges=' + self._request_ranges()
rq = Thread(target=self._request)
rq.setName('HoffmanHTTPDownloader' + rq.getName())
rq.setDaemon(True)
rq.start()
self.active = True
def _request(self):
import encodings.ascii
import encodings.punycode
import encodings.idna
self.error = None
self.received_data = None
try:
self.connection.request('GET', self.url, None, {'User-Agent': VERSION})
r = self.connection.getresponse()
self.connection_status = r.status
self.received_data = r.read()
except Exception as e:
log_exc()
self.error = 'error accessing http seed: ' + str(e)
try:
self.connection.close()
except:
pass
try:
self.connection = HTTPConnection(self.netloc)
except:
self.connection = None
self.downloader.rawserver.add_task(self.request_finished)
def request_finished(self):
self.active = False
if self.error is not None:
if self.goodseed:
self.downloader.errorfunc(self.error)
self.errorcount += 1
if self.received_data:
self.errorcount = 0
if not self._got_data():
self.received_data = None
if not self.received_data:
self._release_requests()
self.downloader.peerdownloader.piece_flunked(self.index)
if self._retry_period:
self.resched(self._retry_period)
self._retry_period = None
return
self.resched()
def _got_data(self):
if self.connection_status == 503:
try:
self.retry_period = max(int(self.received_data), 5)
except:
pass
return False
if self.connection_status != 200:
self.errorcount += 1
return False
self._retry_period = 1
if len(self.received_data) != self.request_size:
if self.goodseed:
self.downloader.errorfunc('corrupt data from http seed - redownloading')
return False
self.measure.update_rate(len(self.received_data))
self.downloader.measurefunc(len(self.received_data))
if self.cancelled:
return False
if not self._fulfill_requests():
return False
if not self.goodseed:
self.goodseed = True
self.downloader.seedsfound += 1
if self.downloader.storage.do_I_have(self.index):
self.downloader.picker.complete(self.index)
self.downloader.peerdownloader.check_complete(self.index)
self.downloader.gotpiecefunc(self.index)
return True
def _get_requests(self):
self.requests = []
self.request_size = 0L
while self.downloader.storage.do_I_have_requests(self.index):
r = self.downloader.storage.new_request(self.index)
self.requests.append(r)
self.request_size += r[1]
self.requests.sort()
def _fulfill_requests(self):
start = 0L
success = True
while self.requests:
begin, length = self.requests.pop(0)
if not self.downloader.storage.piece_came_in(self.index, begin, [], self.received_data[start:start + length], length):
success = False
break
start += length
return success
def _release_requests(self):
for begin, length in self.requests:
self.downloader.storage.request_lost(self.index, begin, length)
self.requests = []
def _request_ranges(self):
s = ''
begin, length = self.requests[0]
for begin1, length1 in self.requests[1:]:
if begin + length == begin1:
length += length1
continue
else:
if s:
s += ','
s += str(begin) + '-' + str(begin + length - 1)
begin, length = begin1, length1
if s:
s += ','
s += str(begin) + '-' + str(begin + length - 1)
return s
def helper_forces_unchoke(self):
pass
def helper_set_freezing(self, val):
self.frozen_by_helper = val
class HTTPRelayClient:
def __init__(self, target):
# Target comes as protocol://target:port/path
self.target = target
_, host, path = target.split(':')
host = host[2:]
self.path = '/' + path.split('/')[1]
self.session = HTTPConnection(host)
self.lastresult = None
def sendNegotiate(self, negotiateMessage):
#Check if server wants auth
self.session.request('GET', self.path)
res = self.session.getresponse()
res.read()
if res.status != 401:
logging.info(
'Status code returned: %d. Authentication does not seem required for URL'
% res.status)
try:
if 'NTLM' not in res.getheader('WWW-Authenticate'):
logging.error(
'NTLM Auth not offered by URL, offered protocols: %s' %
res.getheader('WWW-Authenticate'))
return False
except KeyError:
logging.error(
'No authentication requested by the server for url %s' %
self.target)
return False
#Negotiate auth
negotiate = base64.b64encode(negotiateMessage)
headers = {'Authorization': 'NTLM %s' % negotiate}
self.session.request('GET', self.path, headers=headers)
res = self.session.getresponse()
res.read()
try:
serverChallenge = base64.b64decode(
res.getheader('WWW-Authenticate')[5:])
return serverChallenge
except (IndexError, KeyError):
logging.error('No NTLM challenge returned from server')
def sendAuth(self, authenticateMessageBlob, serverChallenge=None):
#Negotiate auth
auth = base64.b64encode(authenticateMessageBlob)
headers = {'Authorization': 'NTLM %s' % auth}
self.session.request('GET', self.path, headers=headers)
res = self.session.getresponse()
if res.status == 401:
return False
else:
logging.info(
'HTTP server returned error code %d, treating as a succesful login'
% res.status)
#Cache this
self.lastresult = res.read()
return True
#SMB Relay server needs this
@staticmethod
def get_encryption_key():
return None
class TestRegression(TestCase):
def setUp(self):
self.ip = "localhost"
self.port = "6161"
self.headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
self.timeout = 10
# connect
self.conn = HTTPConnection(self.ip, self.port, timeout=self.timeout)
def tearDown(self):
self.conn.close()
def test_1_basic_request(self):
"""
"""
# request an pa computation
path = "/5gt/so/v1/PAComp"
data = {
"ReqId": "string",
"nfvi": {
"resource_types": ["cpu", "ram", "storage", "bandwidth"],
"NFVIPoPs": [{
"id": "A",
"location": {
"center": {
"longitude": 10.878099,
"latitude": 1.9921979
},
"radius": 0
},
"gw_ip_address": "10.10.10.1",
"capabilities": {
"cpu": 100,
"ram": 100,
"storage": 100,
"bandwidth": 100,
"mac": True
},
"availableCapabilities": {
"cpu": 100,
"ram": 100,
"storage": 100,
"bandwidth": 100
},
"failure_rate": 0,
"internal_latency": 2.5
}, {
"id": "B",
"location": {
"center": {
"longitude": 4.878099,
"latitude": 31.9921979
},
"radius": 0
},
"gw_ip_address": "10.10.10.3",
"capabilities": {
"cpu": 100,
"ram": 100,
"storage": 100,
"bandwidth": 100,
"mec": True
},
"availableCapabilities": {
"cpu": 100,
"ram": 100,
"storage": 100,
"bandwidth": 100
},
"failure_rate": 0,
"internal_latency": 3.3
}, {
"id": "C",
"location": {
"center": {
"longitude": -3.878099,
"latitude": 44.9921979
},
"radius": 0
},
"gw_ip_address": "10.10.10.5",
"capabilities": {
"cpu": 100,
"ram": 100,
"storage": 100,
"bandwidth": 100
},
"availableCapabilities": {
"cpu": 100,
"ram": 100,
"storage": 100,
"bandwidth": 100
},
"failure_rate": 0,
"internal_latency": 4.04
}, {
"id": "D",
"location": {
"center": {
"longitude": 5.1426034,
"latitude": 22.0306169
},
"radius": 0
},
"gw_ip_address": "10.10.10.7",
"capabilities": {
"cpu": 100,
"ram": 100,
"storage": 100,
"bandwidth": 100
},
"availableCapabilities": {
"cpu": 100,
"ram": 100,
"storage": 100,
"bandwidth": 100
},
"failure_rate": 0,
"internal_latency": 2.3
}],
"LLs": [{
"LLid": "A-B-1",
"capacity": {
"total": 40,
"available": 40
},
"delay": 9,
"source": {
"id": "A",
"GwIpAddress": "10.10.10.1"
},
"destination": {
"id": "B",
"GwIpAddress": "10.10.10.3"
}
}, {
"LLid": "A-B-2",
"capacity": {
"total": 80,
"available": 80
},
"delay": 5,
"source": {
"id": "A",
"GwIpAddress": "10.10.10.1"
},
"destination": {
"id": "B",
"GwIpAddress": "10.10.10.3"
}
}, {
"LLid": "A-D-1",
"capacity": {
"total": 50,
"available": 50
},
"delay": 12,
"source": {
"id": "A",
"GwIpAddress": "10.10.10.1"
},
"destination": {
"id": "D",
"GwIpAddress": "10.10.10.7"
}
}, {
"LLid": "C-D-1",
"capacity": {
"total": 50,
"available": 50
},
"delay": 11,
"source": {
"id": "C",
"GwIpAddress": "10.10.10.5"
},
"destination": {
"id": "D",
"GwIpAddress": "10.10.10.7"
}
}, {
"LLid": "C-A-1",
"capacity": {
"total": 35,
"available": 55
},
"delay": 3,
"source": {
"id": "C",
"GwIpAddress": "10.10.10.5"
},
"destination": {
"id": "A",
"GwIpAddress": "10.10.10.1"
}
}, {
"LLid": "B-A-1",
"capacity": {
"total": 15,
"available": 45
},
"delay": 12,
"source": {
"id": "B",
"GwIpAddress": "10.10.10.3"
},
"destination": {
"id": "A",
"GwIpAddress": "10.10.10.1"
}
}, {
"LLid": "C-B-1",
"capacity": {
"total": 20,
"available": 20
},
"delay": 5,
"source": {
"id": "C",
"GwIpAddress": "10.10.10.5"
},
"destination": {
"id": "B",
"GwIpAddress": "10.10.10.3"
}
}, {
"LLid": "B-C-1",
"capacity": {
"total": 10,
"available": 60
},
"delay": 7,
"source": {
"id": "B",
"GwIpAddress": "10.10.10.3"
},
"destination": {
"id": "C",
"GwIpAddress": "10.10.10.5"
}
}, {
"LLid": "B-D-1",
"capacity": {
"total": 10,
"available": 70
},
"delay": 7,
"source": {
"id": "B",
"GwIpAddress": "10.10.10.3"
},
"destination": {
"id": "D",
"GwIpAddress": "10.10.10.7"
}
}, {
"LLid": "D-C-1",
"capacity": {
"total": 50,
"available": 50
},
"delay": 7,
"source": {
"id": "D",
"GwIpAddress": "10.10.10.7"
},
"destination": {
"id": "C",
"GwIpAddress": "10.10.10.5"
}
}, {
"LLid": "D-A-1",
"capacity": {
"total": 50,
"available": 50
},
"delay": 7,
"source": {
"id": "D",
"GwIpAddress": "10.10.10.7"
},
"destination": {
"id": "A",
"GwIpAddress": "10.10.10.1"
}
}, {
"LLid": "D-B-1",
"capacity": {
"total": 45,
"available": 45
},
"delay": 3,
"source": {
"id": "D",
"GwIpAddress": "10.10.10.7"
},
"destination": {
"id": "B",
"GwIpAddress": "10.10.10.3"
}
}]
},
"nsd": {
"id":
"rel1-nsd-test",
"name":
"multi-graph-ns",
"SAP": [{
"CPid": "CP1",
"location": {
"center": {
"longitude": -1.0769422,
"latitude": 39.1398602
},
"radius": 20
}
}],
"VNFs": [{
"VNFid":
"v1",
"instances":
1,
"requirements": {
"cpu": 1,
"ram": 100,
"storage": 1
},
"failure_rate":
0,
"processing_latency":
0,
"CP": [{
"cpId": "CP1"
}, {
"cpId": "CP2",
"VNFLink": {
"id": "v1-v2-1",
"latency": 12,
"required_capacity": 30,
"traversal_probability": 0
}
}, {
"cpId": "CP4",
"VNFLink": {
"id": "v1-v2-0",
"latency": 12,
"required_capacity": 10,
"traversal_probability": 0
}
}, {
"cpId": "CP3",
"VNFLink": {
"id": "v1-v4-0",
"latency": 28,
"required_capacity": 5,
"traversal_probability": 0
}
}]
}, {
"VNFid":
"v2",
"instances":
1,
"requirements": {
"cpu": 1,
"ram": 100,
"storage": 1
},
"failure_rate":
0,
"processing_latency":
0,
"CP": [{
"cpId": "CP5",
"VNFLink": {
"id": "v1-v2-0",
"latency": 12,
"required_capacity": 10,
"traversal_probability": 0
}
}, {
"cpId": "CP6",
"VNFLink": {
"id": "v1-v2-1",
"latency": 12,
"required_capacity": 30,
"traversal_probability": 0
}
}, {
"cpId": "CP7",
"VNFLink": {
"id": "v2-v3-0",
"latency": 12,
"required_capacity": 20,
"traversal_probability": 0
}
}]
}, {
"VNFid":
"v3",
"instances":
1,
"requirements": {
"cpu": 1,
"ram": 1,
"storage": 1
},
"failure_rate":
0,
"processing_latency":
0,
"CP": [{
"cpId": "CP8",
"VNFLink": {
"id": "v2-v3-0",
"latency": 12,
"required_capacity": 20,
"traversal_probability": 0
}
}, {
"cpId": "CP9",
"VNFLink": {
"id": "v3-v4-0",
"latency": 40,
"required_capacity": 10,
"traversal_probability": 0
}
}]
}, {
"VNFid":
"v4",
"instances":
1,
"requirements": {
"cpu": 1,
"ram": 1,
"storage": 1,
"mec": True
},
"failure_rate":
0,
"processing_latency":
0,
"CP": [{
"cpId": "CP10",
"VNFLink": {
"id": "v3-v4-0",
"latency": 40,
"required_capacity": 10,
"traversal_probability": 0
}
}, {
"cpId": "CP11",
"VNFLink": {
"id": "v1-v4-0",
"latency": 28,
"required_capacity": 5,
"traversal_probability": 0
}
}]
}],
"VNFLinks": [{
"id": "v1-v2-0",
"latency": 12,
"required_capacity": 10,
"traversal_probability": 0
}, {
"id": "v1-v2-1",
"latency": 12,
"required_capacity": 30,
"traversal_probability": 0
}, {
"id": "v2-v3-0",
"latency": 12,
"required_capacity": 20,
"traversal_probability": 0
}, {
"id": "v3-v4-0",
"latency": 40,
"required_capacity": 10,
"traversal_probability": 0
}, {
"id": "v1-v4-0",
"latency": 28,
"required_capacity": 5,
"traversal_probability": 0
}],
"max_latency":
100,
"target_availability":
0,
"max_cost":
0
},
"callback": "string"
}
self.conn.request("POST", path, dumps(data), self.headers)
self.conn.sock.settimeout(self.timeout)
rsp = self.conn.getresponse()
data = rsp.read()
self.assertEqual(rsp.status, 201)
#print b64data
# if b64data[len(b64data)-2]=='=':
# b64data=b64data[:len(b64data)-2]+'%3d%3d'
# elif b64data[len(b64data)-1]=='=':
# b64data=b64data[:len(b64data)-1]+'%3d'
# params='data='+b64data+'&vcs=8.1.1'
params = s + '&browser=1&vcs=6.1.1'
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent':
'Dalvik/1.6.0 (Linux; U; Android 4.1.2; GT-N7100 Build/JZO54K)',
'Host': Host,
'Connection': 'keep-alive',
'Accept-Encoding': 'gzip',
'Content-Length': len(params),
}
#print params
conn = HTTPConnection(Host)
conn.request('POST', '/ajax/sendChat.php', params, headers)
#conn.close()
#time.sleep(3)
#print i
result = conn.getresponse()
data = result.read()
print data
#f=f+1
#print f
#time.sleep(10)
#res=json.loads(data)
#print res['city_unit_count']['c285']
def post(data):
c = HTTPConnection(ip)
c.request('POST','/servlet/MIMEReceiveServlet',data,{'content-type':'text/xml'})
return BeautifulSoup(c.getresponse().read())
raise err
sleep(0.1)
return None
def kill_server(port, port2server, pids):
try:
kill(pids[port2server[port]], SIGTERM)
except Exception, err:
print err
try_until = time() + 30
while True:
try:
conn = HTTPConnection('127.0.0.1', port)
conn.request('GET', '/')
conn.getresponse()
except Exception, err:
break
if time() > try_until:
raise Exception('Still answering on port %s after 30 seconds' %
port)
sleep(0.1)
def kill_servers(port2server, pids):
for port in port2server:
kill_server(port, port2server, pids)
def kill_nonprimary_server(primary_nodes, port2server, pids):
primary_ports = [n['port'] for n in primary_nodes]
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(self,
host='api.dynect.net',
port=443,
ssl=True,
api_version="current"):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def execute(self, uri, method, args=None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods))
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in headers.items():
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
use_s3 = settings.USE_S3
if use_s3:
s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
bucket = s3.get_bucket(settings.AWS_VIDEO_BUCKET_NAME)
ec2_name = settings.INSTANCE_NAME
tmp_folder = os.path.dirname(os.path.abspath(__file__))
if not use_s3:
tmp_folder = os.path.join(tmp_folder, os.path.pardir, os.path.pardir,
'media', 'tmp')
conn = HTTPConnection(settings.DOMAIN_NAME)
conn.request("GET", "/amazon/ec2-ready/%s/" % settings.INSTANCE_ID)
conn.getresponse()
def encode(ch, method, properties, body):
"""Fetches the temporary video file from s3 and encodes both web and mobile
versions.
Communicates with the screenbird server with the following
- sets the `ec2_node` field of the associated `VideoStatus` for the video
for identification of where to get encoding progress output of ffmpeg
- sets the `web_available` field of the associated `VideoStatus` for the
video after successfully encoding the web version
- sets the `mobile_available` field of the associated `VideoStatus` for the
video after successfully encoding the mobile version
- sets the `video_duration` field of the associated `Video` object
def __call__ (self, host, req):
conn = HTTPConnection (host)
conn.request ("GET", req)
r = conn.getresponse()
return r.read()
class SingleDownload():
def __init__(self, downloader, url):
self.downloader = downloader
self.baseurl = url
try:
(self.scheme, self.netloc, path, pars, query,
fragment) = urlparse(url)
except:
self.downloader.errorfunc('cannot parse http seed address: ' + url)
return
if self.scheme != 'http':
self.downloader.errorfunc('http seed url not http: ' + url)
return
# Arno, 2010-03-08: Make proxy aware
self.proxyhost = find_proxy(url)
try:
if self.proxyhost is None:
self.connection = HTTPConnection(self.netloc)
else:
self.connection = HTTPConnection(self.proxyhost)
except:
self.downloader.errorfunc('cannot connect to http seed: ' + url)
return
self.seedurl = path
if pars:
self.seedurl += ';' + pars
self.seedurl += '?'
if query:
self.seedurl += query + '&'
self.seedurl += 'info_hash=' + urllib.quote(self.downloader.infohash)
self.measure = Measure(downloader.max_rate_period)
self.index = None
self.url = ''
self.requests = []
self.request_size = 0
self.endflag = False
self.error = None
self.retry_period = 30
self._retry_period = None
self.errorcount = 0
self.goodseed = False
self.active = False
self.cancelled = False
self.resched(randint(2, 10))
def resched(self, len=None):
if len is None:
len = self.retry_period
if self.errorcount > 3:
len = len * (self.errorcount - 2)
self.downloader.rawserver.add_task(self.download, len)
def _want(self, index):
if self.endflag:
return self.downloader.storage.do_I_have_requests(index)
else:
return self.downloader.storage.is_unstarted(index)
def download(self):
if DEBUG:
print "http-sdownload: download()"
self.cancelled = False
if self.downloader.picker.am_I_complete():
self.downloader.downloads.remove(self)
return
self.index = self.downloader.picker.next(haveall, self._want, self)
if self.index is None:
self.resched(0.01)
return
if (self.index is None and not self.endflag
and not self.downloader.peerdownloader.has_downloaders()):
self.endflag = True
self.index = self.downloader.picker.next(haveall, self._want, self)
if self.index is None:
self.endflag = True
self.resched()
else:
self.url = (self.seedurl + '&piece=' + str(self.index))
self._get_requests()
if self.request_size < self.downloader.storage._piecelen(
self.index):
self.url += '&ranges=' + self._request_ranges()
rq = Thread(target=self._request)
rq.setName("HoffmanHTTPDownloader" + rq.getName())
rq.setDaemon(True)
rq.start()
self.active = True
def _request(self):
import encodings.ascii
import encodings.punycode
import encodings.idna
self.error = None
self.received_data = None
try:
self.connection.request('GET', self.url, None,
{'User-Agent': VERSION})
r = self.connection.getresponse()
self.connection_status = r.status
self.received_data = r.read()
except Exception, e:
print_exc()
self.error = 'error accessing http seed: ' + str(e)
try:
self.connection.close()
except:
pass
try:
self.connection = HTTPConnection(self.netloc)
except:
self.connection = None # will cause an exception and retry next cycle
self.downloader.rawserver.add_task(self.request_finished)
def encode(ch, method, properties, body):
"""Fetches the temporary video file from s3 and encodes both web and mobile
versions.
Communicates with the screenbird server with the following
- sets the `ec2_node` field of the associated `VideoStatus` for the video
for identification of where to get encoding progress output of ffmpeg
- sets the `web_available` field of the associated `VideoStatus` for the
video after successfully encoding the web version
- sets the `mobile_available` field of the associated `VideoStatus` for the
video after successfully encoding the mobile version
- sets the `video_duration` field of the associated `Video` object
"""
param = eval(body)
slug_tmp = param[0]
host = param[1]
mp4 = os.path.join(tmp_folder, "%s.mp4" % slug_tmp)
slug = slug_tmp.replace("_tmp", "")
text_file = os.path.join(tmp_folder, "%s.txt" % slug)
output_mp4 = os.path.join(tmp_folder, "%s.mp4" % slug)
output_mobile_mp4 = os.path.join(tmp_folder, "%s__mobile.mp4" % slug)
if not use_s3:
shutil.copyfile(output_mp4, mp4)
if use_s3:
key = bucket.get_key(slug_tmp)
else:
key = slug
logger.info("Started encoding for %s" % slug_tmp)
# Define these as variables to make it easier to change later on if need be
set_encoding_url = "/set-encoding-ec2/%s/%s/"
set_available_url = "/set-available/%s/%s/"
time_diff = None
if key:
if use_s3:
# Download source video to be encoded
try:
key.get_contents_to_filename(mp4)
except:
ch.basic_ack(delivery_tag=method.delivery_tag)
# Notify requesting server that video is being processed
try:
conn = HTTPConnection(host)
conn.request("GET", set_encoding_url % (slug, ec2_name))
response = conn.getresponse()
except Exception, e:
pass
logger.info("Encoding for %s - web version" % slug_tmp)
available = False
# Check from requesting server if video is already encoded
try:
conn = HTTPConnection(host)
conn.request("GET", "/is-available/%s/%s/" % (slug, 'web'))
response = conn.getresponse()
available = eval(response.read())
except:
pass
try:
# Start encoding for video if not yet encoded
if not available:
start_time = datetime.datetime.now()
encode_video({
'mp4': str(mp4),
'output_mp4': str(output_mp4),
'text_file': str(text_file)
})
end_time = datetime.datetime.now()
time_diff = end_time - start_time
logger.info("Web encoding time: %s" % time_diff)
else:
logger.info(
"Encoded version for %s already available - web version" %
slug_tmp)
except Exception, e:
logger.info("Encoding for %s failed - web version: %s" %
(slug_tmp, e))
ch.basic_ack(delivery_tag=method.delivery_tag)
ch.basic_publish(exchange='',
routing_key=settings.QUEUE_NAME,
body=body)
publish_interval = 1
pub = Publisher(8290)
server = "rtgpsout.unavco.org:2101"
headers = {
'Ntrip-Version': 'Ntrip/2.0',
'User-Agent': 'NTRIP ntrip_ros',
'Connection': 'close',
'Authorization': 'Basic ' + b64encode(user)
}
connection = HTTPConnection(server)
connection.request('GET', '/SLAC_RTCM3', body=None, headers=headers)
response = connection.getresponse()
# Message format
# https://forum.u-blox.com/index.php/16898/decoding-rtcm3-message
def get_length(msg):
assert (msg[0] == chr(0xd3))
return (ord(msg[1]) * 8 + ord(msg[2])) & 0x3ff
buf = ""
msgs = []
last_message = time.time()
while 1:
def validateURL(url, firstOccurrenceOnly=1, wantRawData=0, groupEvents=0):
"""validate RSS from URL, returns events list, or (events, rawdata) tuple"""
loggedEvents = []
request = urllib2.Request(url)
request.add_header("Accept-encoding", "gzip, deflate")
request.add_header("User-Agent", "FeedValidator/1.3")
usock = None
ctx2 = ssl.create_default_context()
ctx1 = ssl.create_default_context()
ctx2.set_ciphers('ALL:@SECLEVEL=2')
ctx1.set_ciphers('ALL:@SECLEVEL=1')
try:
try:
try:
usock = urllib2.urlopen(request, context=ctx2)
except urllib2.URLError as x:
import re
if True or re.search("WRONG_SIGNATURE_TYPE", x.reason):
loggedEvents.append(
HttpsProtocolWarning(
{'message':
"Weak signature used by HTTPS server"}))
usock = urllib2.urlopen(request, context=ctx1)
else:
raise x
rawdata = usock.read(MAXDATALENGTH)
if usock.read(1):
raise ValidationFailure(
logging.ValidatorLimit({
'limit':
'feed length > ' + str(MAXDATALENGTH) + ' bytes'
}))
# check for temporary redirects
if usock.geturl() != request.get_full_url():
from urlparse import urlsplit
(scheme, netloc, path, query, fragment) = urlsplit(url)
if scheme == 'http':
from httplib import HTTPConnection
requestUri = (path or '/') + (query and '?' + query)
conn = HTTPConnection(netloc)
conn.request("GET", requestUri)
resp = conn.getresponse()
if resp.status != 301:
loggedEvents.append(TempRedirect({}))
except BadStatusLine as status:
raise ValidationFailure(
logging.HttpError({'status': status.__class__}))
except urllib2.HTTPError as status:
rawdata = status.read()
if len(rawdata) < 512 or 'content-encoding' in status.headers:
loggedEvents.append(logging.HttpError({'status': status}))
usock = status
else:
rawdata = re.sub('<!--.*?-->', '', rawdata)
lastline = rawdata.strip().split('\n')[-1].strip()
if sniffPossibleFeed(rawdata):
loggedEvents.append(logging.HttpError({'status': status}))
loggedEvents.append(logging.HttpErrorWithPossibleFeed({}))
usock = status
else:
raise ValidationFailure(
logging.HttpError({'status': status}))
except urllib2.URLError as x:
raise ValidationFailure(logging.HttpError({'status': x.reason}))
except Timeout as x:
raise ValidationFailure(
logging.IOError({
"message": 'Server timed out',
"exception": x
}))
except Exception as x:
raise ValidationFailure(
logging.IOError({
"message": x.__class__.__name__,
"exception": x
}))
if usock.headers.get('content-encoding', None) == None:
loggedEvents.append(Uncompressed({}))
if usock.headers.get('content-encoding', None) == 'gzip':
import gzip, StringIO
try:
rawdata = gzip.GzipFile(
fileobj=StringIO.StringIO(rawdata)).read()
except:
import sys
exctype, value = sys.exc_info()[:2]
event = logging.IOError({
"message":
'Server response declares Content-Encoding: gzip',
"exception": value
})
raise ValidationFailure(event)
if usock.headers.get('content-encoding', None) == 'deflate':
import zlib
try:
rawdata = zlib.decompress(rawdata, -zlib.MAX_WBITS)
except:
import sys
exctype, value = sys.exc_info()[:2]
event = logging.IOError({
"message":
'Server response declares Content-Encoding: deflate',
"exception": value
})
raise ValidationFailure(event)
if usock.headers.get('content-type',
None) == 'application/vnd.google-earth.kmz':
import tempfile, zipfile, os
try:
(fd, tempname) = tempfile.mkstemp()
os.write(fd, rawdata)
os.close(fd)
zfd = zipfile.ZipFile(tempname)
namelist = zfd.namelist()
for name in namelist:
if name.endswith('.kml'):
rawdata = zfd.read(name)
zfd.close()
os.unlink(tempname)
except:
import sys
value = sys.exc_info()[:1]
event = logging.IOError({
"message": 'Problem decoding KMZ',
"exception": value
})
raise ValidationFailure(event)
mediaType = None
charset = None
# Is the Content-Type correct?
contentType = usock.headers.get('content-type', None)
if contentType:
(mediaType,
charset) = mediaTypes.checkValid(contentType, loggedEvents)
# Check for malformed HTTP headers
for (h, v) in usock.headers.items():
if (h.find(' ') >= 0):
loggedEvents.append(HttpProtocolError({'header': h}))
selfURIs = [request.get_full_url()]
baseURI = usock.geturl()
if not baseURI in selfURIs: selfURIs.append(baseURI)
# Get baseURI from content-location and/or redirect information
if usock.headers.get('content-location', None):
from urlparse import urljoin
baseURI = urljoin(baseURI,
usock.headers.get('content-location', ""))
elif usock.headers.get('location', None):
from urlparse import urljoin
baseURI = urljoin(baseURI, usock.headers.get('location', ""))
if not baseURI in selfURIs: selfURIs.append(baseURI)
usock.close()
usock = None
mediaTypes.contentSniffing(mediaType, rawdata, loggedEvents)
encoding, rawdata = xmlEncoding.decode(mediaType,
charset,
rawdata,
loggedEvents,
fallback='utf-8')
if rawdata is None:
return {'loggedEvents': loggedEvents}
rawdata = rawdata.replace('\r\n', '\n').replace('\r',
'\n') # normalize EOL
validator = _validate(rawdata,
firstOccurrenceOnly,
loggedEvents,
baseURI,
encoding,
selfURIs,
mediaType=mediaType,
groupEvents=groupEvents)
# Warn about mismatches between media type and feed version
if mediaType and validator.feedType:
mediaTypes.checkAgainstFeedType(mediaType, validator.feedType,
validator.loggedEvents)
params = {
"feedType": validator.feedType,
"loggedEvents": validator.loggedEvents
}
if wantRawData:
params['rawdata'] = rawdata
return params
finally:
try:
if usock: usock.close()
except:
pass
