def health_check(self, target):
global dead_servers
is_alive = 0
beaten = 0
p = re.compile("[a-z0-9-.]*?\.[a-z]+$")
domain_tld = p.findall(target)[0]
accepted_responses = (404, 302, 304, 301, 200)
#logger.error('perform health check on ' + domain_tld)
try:
conn = HTTPConnection(domain_tld)
conn.request('HEAD', '/')
response = conn.getresponse()
if response.status in accepted_responses:
is_alive = 1
else:
logger.error(ctime() + ': ' + target + ' is down, HTTP error code is: ' + response.status )
# FIXME
# Bypass dns resolving errors exception, don't know why it always fail after running for about 1 hours.
except gaierror:
#logger.error(ctime() + ' ' + target + ' : Error: Name or service does not known' )
is_alive = 1
except error:
logger.error(ctime() + ' ' + target + ' : Connection refused' )
finally:
conn.close()
if is_alive == 0 and target not in dead_servers:
dead_servers.append(target)
if is_alive == 1 and target in dead_servers:
dead_servers.pop(dead_servers.index(target))
def _make_request(method, host, path, body='', headers={}):
"""
Makes an HTTP request.
Args:
method |str| = Type of request. GET, POST, PUT or DELETE.
host |str| = The host and port of the server. E.g. vmpooler.myhost.com:8080
path |str| = The path of the url. E.g. /vm/vm_name
body |str| = The body data to send with the request.
headers |{str:str}| = Optional headers for the request.
Returns:
|HTTPResponse| = Response from the request.
Raises:
|RuntimeError| = If the vmpooler URL can't be reached
"""
try:
conn = HTTPConnection(host)
conn.request(method, path, body, headers)
resp = conn.getresponse()
return resp
except gaierror:
error = "Couldn't connect to address '{}'. Ensure this is the correct URL for " \
"the vmpooler".format(host)
raise RuntimeError(error)
except:
print("Unkown error occured while trying to connect to {}".format(host))
raise
def run(self):
conn = HTTPConnection(self.proxy)
tmpFlag = True
dataFile = open(self.file, "r")
cdata = dataFile.read()
dataFile = open(self.file2, "r")
cdata2 = dataFile.read()
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if rdata != cdata:
tmpFlag = False
if resp.will_close == True:
tmpFlag = False
connHdrs = {"Connection": "close"}
conn.request("GET", self.url2, headers=connHdrs)
resp = conn.getresponse()
rdata2 = resp.read()
if rdata2 != cdata2:
tmpFlag = False
if resp.will_close == False:
tmpFlag = False
if tmpFlag == True:
self.result = True
conn.close()
dataFile.close()
def run(self):
if self.file:
dataFile = open(self.file, "r")
cdata = dataFile.read()
conn = HTTPConnection(self.proxy)
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if rdata == cdata:
self.result = True
self.data = rdata
conn.close()
dataFile.close()
else:
conn = HTTPConnection(self.proxy)
conn.request("GET", self.url)
resp = conn.getresponse()
rdata = resp.read()
if resp.status == httplib.OK:
self.result = True
conn.close()
def proxy(request):
if 'url' not in request.GET:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain"
)
url = urlsplit(request.GET['url'])
# Don't allow localhost connections unless in DEBUG mode
if not settings.DEBUG and re.search('localhost|127.0.0.1', url.hostname):
return HttpResponse(status=403)
locator = url.path
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
# Strip all headers and cookie info
headers = {}
conn = HTTPConnection(url.hostname, url.port) if url.scheme == "http" else HTTPSConnection(url.hostname, url.port)
conn.request(request.method, locator, request.raw_post_data, headers)
result = conn.getresponse()
response = HttpResponse(
valid_response(result.read()),
status=result.status,
content_type=result.getheader("Content-Type", "text/plain")
)
return response
def _select(self, params):
# encode the query as utf-8 so urlencode can handle it
params['q'] = unicode_safe(params['q'])
path = '%s/select/?%s' % (self.path, urlencode(params))
conn = HTTPConnection(self.host, self.port)
conn.request('GET', path)
return conn.getresponse()
def testGet(self):
self.assertTrue(self.pasteThread.isAlive())
from httplib import HTTPConnection
http_connection = HTTPConnection("localhost", port=self.port)
http_connection.request("GET", "/api/Crawl")
response = http_connection.getresponse()
self.assertTrue(response.status == 404 or response.status == 200 or response.status == 500)
def send(sms_to, sms_body, **kwargs):
"""
Site: http://iqsms.ru/
API: http://iqsms.ru/api/
"""
headers = {
"User-Agent": "DBMail/%s" % get_version(),
'Authorization': 'Basic %s' % b64encode(
"%s:%s" % (
settings.IQSMS_API_LOGIN, settings.IQSMS_API_PASSWORD
)).decode("ascii")
}
kwargs.update({
'phone': sms_to,
'text': from_unicode(sms_body),
'sender': kwargs.pop('sms_from', settings.IQSMS_FROM)
})
http = HTTPConnection(kwargs.pop("api_url", "gate.iqsms.ru"))
http.request("GET", "/send/?" + urlencode(kwargs), headers=headers)
response = http.getresponse()
if response.status != 200:
raise IQSMSError(response.reason)
body = response.read().strip()
if '=accepted' not in body:
raise IQSMSError(body)
return int(body.split('=')[0])
def get_embed(video_url):
if video_url.startswith('http://www.youtube.com/watch?v='):
yid = video_url.partition('=')[2]
return 'http://www.youtube.com/embed/%(yid)s?wmode=opaque&fs=1&theme=dark&autoplay=0&' % {'yid':yid}
conn = HTTPConnection("video.yandex.net")
get_embed_url = "/embed.xml?params=%26embed%3Dy&width=612.8&height=344.7&link=" + quote(video_url)
logger.info("Getting embed: " + get_embed_url)
conn.request("GET",get_embed_url)
res = conn.getresponse()
result = res.getheader("Location")
if result is not None:
result = result.replace("?autoplay=1","?autoplay=0")
result = result.replace("&autoplay=1","&autoplay=0")
result = result.replace("?autoStart=true","?autoStart=false")
result = result.replace("&autoStart=true","&autoStart=false")
result = result.replace("?auto=1","?auto=0")
result = result.replace("&auto=1","&auto=0")
result = result.replace("?ap=1","?ap=0")
result = result.replace("&ap=1","&ap=0")
if 'video.yandex.ru' in result:
result = result.replace("&autoplay=0","")
result = result.replace("?autoplay=0&","?")
result = result.replace("?autoplay=0","")
return result
def send(sms_to, sms_body, **kwargs):
"""
Site: http://smsaero.ru/
API: http://smsaero.ru/api/
"""
headers = {
"User-Agent": "DBMail/%s" % get_version(),
}
kwargs.update({
'user': settings.SMSAERO_LOGIN,
'password': settings.SMSAERO_MD5_PASSWORD,
'from': kwargs.pop('sms_from', settings.SMSAERO_FROM),
'to': sms_to.replace('+', ''),
'text': from_unicode(sms_body)
})
http = HTTPConnection(kwargs.pop("api_url", "gate.smsaero.ru"))
http.request("GET", "/send/?" + urlencode(kwargs), headers=headers)
response = http.getresponse()
if response.status != 200:
raise AeroSmsError(response.reason)
body = response.read().strip()
if '=accepted' not in body:
raise AeroSmsError(body)
return int(body.split('=')[0])
def check_url_path(path, redirected=0):
if redirected > MAX_REDIRECTION_ALLOWED:
return False
try:
code = ''
parse_result = urlparse(path)
server_name = parse_result.netloc
urlpath = parse_result.path
if not urlpath:
# Just a server, as with a repo.
with contextlib.closing(urllib2.urlopen(path)) as res:
code = res.getcode()
else:
# socket.gaierror could be raised,
# which is a child class of IOError
conn = HTTPConnection(server_name, timeout=15)
# Don't try to get the whole file:
conn.request('HEAD', path)
response = conn.getresponse()
code = response.status
conn.close()
if code == 200:
return True
elif code == 301 or code == 302:
for header in response.getheaders():
if header[0] == 'location':
return check_url_path(header[1], redirected+1)
else:
return False
except (urllib2.URLError, HTTPException, IOError, ValueError):
return False
return True
def api(api_name, path):
c = membase.incr(request.environ['REMOTE_ADDR'])
if c == None:
membase.set(request.environ['REMOTE_ADDR'], 1, 86400)
elif c > 50000:
abort(403)
apis = {'silcc': 'www.opensilcc.com'}
path = '/' + path
if '?' in request.url:
path += '?' + request.url.split('?')[1]
api = HTTPConnection(apis[api_name], strict=True)
api.connect()
api.request(request.method, path, request.data)
api_response = api.getresponse()
api_response_content = api_response.read()
api.close()
gateway_response = make_response(api_response_content)
gateway_response.status_code = api_response.status
for header in ['Cache-Control', 'Content-Type', 'Pragma']:
gateway_response.headers[header] = api_response.getheader(header)
log_data = dict(api=api_name, path=path, data=request.data, response=api_response_content)
log_entry = json.dumps(log_data)
pika_channel.basic_publish(exchange='', routing_key='swiftgate', body=log_entry)
return gateway_response
def _httplib_request(self, method, url, fmt, params):
'''Make a HTTP request via httplib.
Make a HTTP request to `url` using `method`. `fmt` denotes a desired
response format (sent as ``Accept:`` header), `params` are sent as
the query string or the request body, depending on the method.
Return a `httplib.HTTPResponse` instance.
'''
methods = ('DELETE', 'GET', 'POST', 'PUT')
method = method.upper()
if not method in methods:
raise MyGengoException('Invalid or unsupported HTTP method %s' % method)
query_string = urlencode(params)
url_parts = urlparse(url)
path = url_parts.path
if method in ('POST', 'PUT'):
body = query_string
else:
path += '?' + query_string
body = None
if self.config.debug:
print url_parts.hostname + path
headers = self._make_header(fmt)
if method == 'POST':
headers['Content-Type'] = 'application/x-www-form-urlencoded'
try:
conn = HTTPConnection(url_parts.hostname, url_parts.port)
headers['User-Agent'] = 'pythonMyGengo 1.0'
conn.request(method, path, body, headers=headers)
return conn.getresponse()
except Exception, e:
raise MyGengoException(e.args)
def getSuggestions(self, queryString):
if not queryString:
return None
else:
query = '/complete/search?output=toolbar&client=youtube&xml=true&ds=yt'
if self.gl:
query += '&gl=' + self.gl
if self.hl:
query += '&hl=' + self.hl
query += '&jsonp=self.getSuggestions&q=' + quote(queryString)
try:
connection = HTTPConnection('google.com')
connection.request('GET', query, '', {'Accept-Encoding': 'UTF-8'})
except (CannotSendRequest, gaierror, error):
print "[YouTube] Can not send request for suggestions"
else:
try:
response = connection.getresponse()
except BadStatusLine:
print "[YouTube] Can not get a response from google"
else:
if response.status == 200:
data = response.read()
try:
charset = response.getheader('Content-Type',
'text/xml; charset=ISO-8859-1').rsplit('=')[1]
except:
charset = 'ISO-8859-1'
connection.close()
return data.decode(charset).encode('utf-8')
if connection:
connection.close()
return None
def _publish(args, server, uri):
from httplib import HTTPConnection
from urllib import urlencode
args = dict((k,v) for k,v in args.items() if v != 'NA')
uri = uri + "?" + urlencode(args)
log.debug('Connect to: http://%s' % server)
log.debug('GET %s' % uri)
conn = HTTPConnection(server)
if not conn:
raise PublishException('Remote server connection error')
conn.request("GET", uri)
try:
http = conn.getresponse()
data = (http.status, http.reason, http.read())
conn.close()
if not (data[0] == 200 and data[1] == 'OK'):
raise PublishException('Server returned invalid status: %d %s %s'
% data)
except (DownloadError):
logging.warning("Download error: ", sys.exc_info()[0])
data = None
return data
def create(self, query, type, feed=True,
freq=ALERT_FREQS[FREQ_AS_IT_HAPPENS]):
"""
Creates a new alert.
:param query: the search terms the alert will match
:param type: a value in :attr:`ALERT_TYPES` indicating the desired
results
:param feed: whether to deliver results via feed or email
:param freq: a value in :attr:`ALERT_FREQS` indicating how often results
should be delivered; used only for email alerts (feed alerts are
updated in real time)
"""
headers = {'Cookie': self.cookie,
'Content-type': 'application/x-www-form-urlencoded; charset=utf-8'}
params = safe_urlencode({
'q': query,
'e': DELIVER_FEED if feed else self.email,
'f': ALERT_FREQS[FREQ_AS_IT_HAPPENS] if feed else freq,
't': ALERT_TYPES[type],
'sig': self._scrape_sig(),
})
conn = HTTPConnection('www.google.com')
conn.request('POST', '/alerts/create?hl=en&gl=us', params, headers)
response = conn.getresponse()
try:
if response.status != 302:
raise UnexpectedResponseError(response.status, response.getheaders(), response.read())
finally:
conn.close()
def send_email(request):
try:
recipients = request.GET['to'].split(',')
url = request.GET['url']
proto, server, path, query, frag = urlsplit(url)
if query: path += '?' + query
conn = HTTPConnection(server)
conn.request('GET',path)
resp = conn.getresponse()
assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason)
rawData = resp.read()
conn.close()
message = MIMEMultipart()
message['Subject'] = "Graphite Image"
message['To'] = ', '.join(recipients)
message['From'] = 'composer@%s' % gethostname()
text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(),url) )
image = MIMEImage( rawData )
image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png"))
message.attach(text)
message.attach(image)
s = SMTP(settings.SMTP_SERVER)
s.sendmail('composer@%s' % gethostname(),recipients,message.as_string())
s.quit()
return HttpResponse( "OK" )
except:
return HttpResponse( format_exc() )
def request(self, method, endpoint, qargs={}, data=None):
path = self.format_path(endpoint, qargs)
conn = HTTPConnection('%s:80' % self.account)
auth = b64encode("%s:%s" % (self.api_key, self.password))
headers = {
'Authorization': 'Basic %s' % auth,
'Content-Type': 'application/xml'
}
done = False
while not done:
try:
conn.request(method, path, headers=headers, body=data)
except socket.gaierror:
if self.retry_on_socket_error:
time.sleep(self.retry_timeout)
continue
else:
raise
resp = conn.getresponse()
body = resp.read()
if resp.status == 503 and self.retry_on_503:
time.sleep(self.retry_timeout)
else:
done = True
if 200 <= resp.status < 300:
return body
else:
raise ApiError("%s request to %s returned: %s\n%s" %
(method, path, resp.status, body), resp.status)
class HTTPSpeakerClient:
"""Emacspeak HTTP speech client, for HTTPSpeaker instances."""
def __init__(self, host="127.0.0.1", port=8000):
"Initialize client to connect to server at given host and port."
self._connection=HTTPConnection(host, port)
def postCommand(self, command, arg=""):
"""Post command, with argument arg (default, empty), to the speech server.
Returns the body of the server's HTTP response, if any. On error, HTTPSpeakerError is raised."""
body = command
if arg:
body += ": " + arg
self._connection.request("POST", "/", body, {"Content-type": "text/plain"})
response=self._connection.getresponse()
if response.status != 200:
raise HTTPSpeakerError(response.status, response.reason)
return response.read()
def speak(self, text):
"Speak the supplied string."
self.postCommand("speak", text)
def stop(self):
"Stop speaking."
self.postCommand("stop")
def isSpeaking(self):
"Return '0' when not speaking."
return self.postCommand("isSpeaking")
def close(self):
"Close the connection to the speech server."
self._connection.close()
def search_on_playdeb(orig_file):
global MIRROR_URL, GETDEB_SUBDIR
http_connection = HTTPConnection(MIRROR_URL)
basename = orig_file.split('_')[0]
download_dir = 'http://' + MIRROR_URL + '/' + GETDEB_SUBDIR + '/ubuntu/pool/games/' + get_package_subdir(orig_file) + \
'/' + basename + '/'
http_connection.request('GET', download_dir)
http_response = http_connection.getresponse()
if http_response.status != 200: return None
data = http_response.read()
http_connection.close()
data = data.split('\n')
package_lines = list()
for line in data:
if basename in line:
package_lines.append(line)
if len(package_lines) == 0: return None
p_d = list()
package_re = re.compile('<a .*?>(?P<orig>.*?)(?:\.diff\.gz|\.debian\.tar\.gz)<')
download_re = re.compile('<a href="(?P<download>.*?)">')
for line in package_lines:
search_result = re.search(package_re, line)
if not search_result: continue
orig = search_result.group('orig')
search_result = re.search(download_re, line)
download = download_dir + search_result.group('download')
p_d.append((orig,download))
return p_d
class RESTClient(object):
"""
HTTP client
HTTPS client based on the provided URL (http:// or https://)
"""
def __init__(self, url, cert=None, key=None):
logging.info("RESTClient URL: %s" % url)
if url.startswith("https://"):
logging.info("Using HTTPS protocol, getting user identity files ...")
proxy_file = "/tmp/x509up_u%s" % os.getuid()
if not os.path.exists(proxy_file):
proxy_file = "UNDEFINED"
cert_file = cert or os.getenv("X509_USER_CERT", os.getenv("X509_USER_PROXY", proxy_file))
key_file = key or os.getenv("X509_USER_KEY", os.getenv("X509_USER_PROXY", proxy_file))
logging.info("Identity files:\n\tcert file: '%s'\n\tkey file: '%s' " % (cert_file, key_file))
url = url.replace("https://", "")
logging.info("Creating connection HTTPS ...")
self.conn = HTTPSConnection(url, key_file=key_file, cert_file=cert_file)
if url.startswith("http://"):
logging.info("Using HTTP protocol, creating HTTP connection ...")
url = url.replace("http://", "")
self.conn = HTTPConnection(url)
def http_request(self, verb, uri, data=None, header=None):
logging.debug("Request: %s %s %s ..." % (verb, uri, data))
self.conn.request(verb, uri, body=data, headers=self.headers)
resp = self.conn.getresponse()
data = resp.read()
logging.debug("Status: %s" % resp.status)
logging.debug("Reason: %s" % resp.reason)
return resp.status, data
def send_youtube_information(msg, sock):
global channels
matches = re.finditer('youtube\.com(?P<link>/watch\S*)', msg)
if not matches: return
for match in matches:
http_connection = HTTPConnection('www.youtube.com')
http_connection.request('GET', match.group('link'))
http_response = http_connection.getresponse()
if http_response.status != 200:
print "Error occured when fetching data"
continue
data = http_response.read(4096)
titles = re.finditer('<title>(?P<title>.*)</title>', data, re.DOTALL)
for title in titles:
video_title = title.group('title')
video_title = video_title.split('-', 1)
video_title = video_title[1].strip()
msg = "PRIVMSG " + channels + u" :\u0002" + video_title + \
u"\u000F www.youtube.com" + match.group('link') + "\r\n"
sock.send(msg)
http_connection.close()
return
def proxy(request):
if 'url' not in request.GET:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain"
)
url = urlsplit(request.GET['url'])
locator = url.path
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
headers = {}
if settings.SESSION_COOKIE_NAME in request.COOKIES:
headers["Cookie"] = request.META["HTTP_COOKIE"]
if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
headers["Content-Type"] = request.META["CONTENT_TYPE"]
conn = HTTPConnection(url.hostname, url.port)
conn.request(request.method, locator, request.raw_post_data, headers)
result = conn.getresponse()
response = HttpResponse(
result.read(),
status=result.status,
content_type=result.getheader("Content-Type", "text/plain")
)
return response
def submit():
# Build the parameters
headers = {"Content-type": "application/x-www-form-urlencoded"}
info = urlencode(
{
"version": configured.VERSION,
"comments": CTK.post["comments"],
"traceback": unquote(CTK.post["traceback"]),
"config": str(CTK.cfg),
}
)
# HTTP Request
conn = HTTPConnection(URL_REPORT_HOST)
conn.request("POST", URL_REPORT_URL, info, headers)
response = conn.getresponse()
data = response.read()
conn.close()
# Check the output
if response.status != 200:
return {"ret": "error"}
if not "{'ret':'ok'}" in data:
return {"ret": "error"}
return {"ret": "ok"}
def commit_koalemos(r):
conn = HTTPConnection(OPTIONS['host'], OPTIONS['port'])
conn.request('POST', '/commit',
body=urlencode({'r': r}),
headers={'Content-Type': 'application/x-www-form-urlencoded'})
res = conn.getresponse()
return (None, False) if res.status == 204 else (res.read(), True)
def update(self, alert):
"""
Updates an existing alert which has been modified.
"""
headers = {'Cookie': self.cookie,
'Content-type': 'application/x-www-form-urlencoded; charset=utf-8'}
sig, es, hps = self._scrape_sig_es_hps(alert)
params = {
'd': DELIVER_TYPES.get(alert.deliver, DELIVER_DEFAULT_VAL),
'e': self.email,
'es': es,
'hps': hps,
'q': alert.query,
'se': 'Save',
'sig': sig,
't': ALERT_TYPES[alert.type],
}
if alert.deliver == DELIVER_EMAIL:
params['f'] = ALERT_FREQS[alert.freq]
params = safe_urlencode(params)
conn = HTTPConnection('www.google.com')
conn.request('POST', '/alerts/save?hl=en&gl=us', params, headers)
response = conn.getresponse()
try:
if response.status != 302:
raise UnexpectedResponseError(response.status, response.getheaders(), response.read())
finally:
conn.close()
def run(query='insurance', query_template_file='query.txt', host='localhost',
port='8953', interval='1.0', timeout='10'):
with open(join(dirname(__file__), query_template_file)) as fh:
url = string.Template(fh.read()).substitute(
host=host,
port=port,
query=quote_plus(query))
print 'Starting load'
con = HTTPConnection(host, port, timeout=float(timeout))
while True:
try:
con.request('GET', url)
sys.stdout.write('.')
sys.stdout.flush()
response = con.getresponse()
content = response.read()
assert 'HTTP ERROR: 500' not in content, 'Server error %s' % content
time.sleep(float(interval))
except KeyboardInterrupt:
break
print '\nEnd'
def report_sauce_status(job_id, test_status):
"""Report test status to Sauce service
:param string job_id: [required] saucelabs job id
:param string test_status: [required] status of test
:returns: request status code
:rtype: int or string
Example::
Report sauce status ${SESSION_ID} ${TEST_STATUS}
"""
username = environ.get('SAUCE_USERNAME')
access_key = environ.get('SAUCE_ACCESS_KEY')
if not job_id:
return u"No Sauce job id found. Skipping..."
elif not username or not access_key:
return u"No Sauce environment variables found. Skipping..."
token = encodestring('%s:%s' % (username, access_key))[:-1]
body = dumps({'passed': test_status == 'PASS'})
connection = HTTPConnection('saucelabs.com')
connection.request('PUT', '/rest/v1/%s/jobs/%s' % (
username, job_id), body,
headers={'Authorization': 'Basic %s' % token}
)
return connection.getresponse().status
def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT):
server = port2server[port]
if server[:-1] in ('account', 'container', 'object'):
if int(server[-1]) > 4:
return None
path = '/connect/1/2'
if server[:-1] == 'container':
path += '/3'
elif server[:-1] == 'object':
path += '/3/4'
try_until = time() + timeout
while True:
try:
conn = HTTPConnection('127.0.0.1', port)
conn.request('GET', path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception, err:
if time() > try_until:
print err
print 'Giving up on %s:%s after %s seconds.' % (
server, port, timeout)
raise err
sleep(0.1)
class GoodReadsBookMixin(object):
def __init__(self):
self.http = HTTPConnection("www.goodreads.com")
self.stats = None
if self.id is not None and self._goodreads_id == 0:
self._get_goodreads_stats()
def _get_goodreads_stats(self):
"""
{
u'work_ratings_count': 4129,
u'isbn': u'0767905385',
u'text_reviews_count': 98,
u'isbn13': u'9780767905381',
u'reviews_count': 932,
u'work_reviews_count': 6445,
u'average_rating': u'3.83',
u'work_text_reviews_count': 537,
u'ratings_count': 529,
u'id': 375789
}
"""
url = "/book/review_counts.json?isbns=%s&key=%s" % (self.isbn13, settings.GOODREADS_KEY)
self.http.request('GET', url)
json = self.http.getresponse().read()
try:
data = simplejson.loads(json)
except ValueError, e:
self._goodreads_id = -1
self.save()
return
self.stats = data["books"][0]
def _send_request(self, method, path, body=None, headers=None):
if TIMEOUTS_AVAILABLE:
http = Http(timeout=self.timeout)
url = self.base_url + path
try:
start_time = time.time()
self.log.debug(
"Starting request to '%s' (%s) with body '%s'..." %
(url, method, str(body)[:10]))
headers, response = http.request(url,
method=method,
body=body,
headers=headers)
end_time = time.time()
self.log.info(
"Finished '%s' (%s) with body '%s' in %0.3f seconds." %
(url, method, str(body)[:10], end_time - start_time))
except AttributeError:
# For httplib2.
error_message = "Failed to connect to server at '%s'. Are you sure '%s' is correct? Checking it in a browser might help..." % (
url, self.base_url)
self.log.error(error_message)
raise SolrError(error_message)
if int(headers['status']) != 200:
error_message = self._extract_error(headers, response)
self.log.error(error_message)
raise SolrError(error_message)
return response
else:
if headers is None:
headers = {}
conn = HTTPConnection(self.host, self.port)
start_time = time.time()
self.log.debug(
"Starting request to '%s:%s/%s' (%s) with body '%s'..." %
(self.host, self.port, path, method, str(body)[:10]))
conn.request(method, path, body, headers)
response = conn.getresponse()
end_time = time.time()
self.log.info(
"Finished '%s:%s/%s' (%s) with body '%s' in %0.3f seconds." %
(self.host, self.port, path, method, str(body)[:10],
end_time - start_time))
if response.status != 200:
error_message = self._extract_error(
dict(response.getheaders()), response.read())
self.log.error(error_message)
raise SolrError(error_message)
return response.read()
def upload_file(self):
f = open(self.filename, "r")
data = f.read()
f.close()
data = data.replace("CALLBACK_IP", self.callback.ip)
data = data.replace("CALLBACK_PORT", str(self.callback.port))
try:
if self.https == 0:
cnx = HTTPConnection(self.host, self.port)
else:
cnx = HTTPSConnection(self.host, self.port)
# Upload JSP file
url = "/robohelp/server?PUBLISH=" + str(random.randint(1, 100))
headers = {
'User-Agent':
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
'Content-Type':
'multipart/form-data; boundary=---------------------------111',
'UID': '1234'
}
param = "-----------------------------D2BOUND\r\n"
param += "Content-Disposition: form-data; name=\"filename\"; filename=\"d2.jsp\"\r\n"
param += "Content-Type: application/x-java-archive\r\n\r\n"
param += data
param += "\r\n"
param += "-----------------------------D2BOUND--\r\n"
cnx.request("POST", url, param, headers)
resp = cnx.getresponse()
hdr = resp.getheaders()
for h in hdr:
if (h[0] == "sessionid"):
id = h[1]
# Execute uploaded JSP file
url = "/robohelp/robo/reserved/web/%s/d2.jsp" % (id)
cnx.request(
"GET", url, None, {
'User-Agent':
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)'
})
cnx.close()
except:
self.log("HTTP(S) error")
return 0
return 1
def get_response(url):
'''Return response object from URL'''
try:
conn = HTTPConnection(url)
conn.request('HEAD', '/')
return conn.getresponse()
except socket.error:
return None
except:
logging.error('Bad URL:', url)
exit(1)
def get_login(host, port, name, pwd):
conn = HTTPConnection(host, port)
conn.request('GET', '/login/%s/%s' % (name, pwd))
res = conn.getresponse()
if res.status != 200:
return
data = res.read()
firstline = data.split('\n')[0]
name, value = firstline.split('=')
assert name == 'port'
return int(value)
def response(context, flow):
if flow.request.host.endswith("nianticlabs.com"):
with decoded(flow.response):
conn = HTTPConnection("127.0.0.1")
conn.request("POST", "/api/update", flow.response.content)
res = conn.getresponse()
if res.status / 100 not in [2, 3]:
context.log("Got error sending mitm data to api ({})".format(
res.status),
level="error")
def get_length(self):
"""
"""
conn = HTTPConnection(self.host)
conn.request('GET', self.rest, headers={'Range': '0-1'})
length = int(conn.getresponse().getheader('content-length'))
if self.verbose:
print >> stderr, length, 'bytes in', basename(self.rest)
return length
class CommandLineFu(object):
def __init__(self):
self._h = HTTP(API_DOMAIN)
def _operate(self, action, search_limit):
index = 0
search_result = []
while True:
if search_limit != None and index > search_limit:
break
self._h.request("GET", action)
index = index + 25
res = loads(self._h.getresponse().read())
if res == []:
break
else: search_result += res
return search_result
def browse(self, sort_order=SORT_BY_DATES, index = 0):
"""
Browse Commandlinefu history, sorted by dates or by votes.
By default, CLF has 25 results per page, so index can be used
to see others 25 results.
"""
self._h.request("GET","/commands/browse/%s/json/%d"%(sort_order,index))
return self._h.getresponse().read()
def matching(self, match, sort_order = SORT_BY_DATES, search_limit = None):
"""
Search from history commands that match the "match" string.
By default, it returns all result, sort by dates with no search limit.
"""
match64 = encodestring(match)[:-1]
action = "/commands/matching/%s/%s/%s/json/%d"%(match, match64, sort_order, index)
return self._operate(action, search_limit)
def using(self, match, sort_order = SORT_BY_DATES, search_limit = None):
"""
Returns commands that implements the search word "match'.
By default, it returns all result, sort by dates with no search limit.
"""
action = "/commands/using/%s/%s/json/%d"%(match, sort_order, index)
return self._operate(action, search_limit)
def by(self, match, sort_order = SORT_BY_DATES, search_limit = None):
"""
Returns commands from a specified user.
By default, it returns all result, sort by dates with no search limit.
"""
action = "/commands/by/%s/%s/json/%d"%(match, sort_order, index)
return self._operate(action, search_limit)
def url_ok(url, port):
from httplib import HTTPConnection
try:
conn = HTTPConnection(url, port)
conn.request("GET", "/")
r = conn.getresponse()
return r.status == 200
except:
logger.exception("Server not started")
return False
def resolveUrl(url):
"""Follows a redirect one level and returns the location of the HTTP 302 redirect"""
url = urlparse.urlparse(url)
connection = HTTPConnection(url[1])
connection.request("GET", url[2])
request = connection.getresponse()
if request.status == 302: # header "Found:", might need more here
realUrl = request.getheader("Location")
else:
realUrl = urlparse.urlunparse(url)
return realUrl
def test_response_body(webapp):
connection = HTTPConnection(webapp.server.host, webapp.server.port)
connection.connect()
connection.request("GET", "/response_body")
response = connection.getresponse()
assert response.status == 200
assert response.reason == "OK"
s = response.read()
assert s == u"ä".encode('utf-8')
connection.close()
def CheckInet():
con = HTTPConnection('www.example.com')
try:
con.request('HEAD', '/')
return 'OK'
except:
Quit('[!] Internet NOT Available')
# install squid, may require internet connection
''' DEBIAN ONLY '''
''' CHANGE THIS TO SUIT YOUR OS' INSTALLATION INSTRUCTION/COMMAND '''
def short_url(href):
'''
'''
conn = HTTPConnection('teczno.com', 80)
conn.request('POST', '/s/', urlencode({'url': href}),
{'Content-Type': 'application/x-www-form-urlencoded'})
resp = conn.getresponse()
body = resp.read()
assert resp.status in range(200, 299), (resp.status, body)
return body.strip()
def test_request_body(webapp, body):
connection = HTTPConnection(webapp.server.host, webapp.server.port)
connection.connect()
connection.request("GET", "/request_body", body)
response = connection.getresponse()
assert response.status == 200
assert response.reason == "OK"
s = response.read()
assert s == body
connection.close()
def json_rest_call(self, __in_json_data, __in_s_action, __in_s_path):
"""json rest call function."""
s_headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
json_msg_body = json.dumps(__in_json_data)
http_connect = HTTPConnection(self.s_controller_ip,
self.i_controller_port)
try:
http_connect.request(__in_s_action, __in_s_path, json_msg_body,
s_headers)
http_response = http_connect.getresponse()
rest_response = (http_response.status, http_response.reason,
http_response.read())
http_connect.close()
except httplib.HTTPException as error:
print(str(error))
return None
except httplib.NotConnected as error:
print(str(error))
return None
except httplib.InvalidURL as error:
print(str(error))
return None
except httplib.UnknownProtocol as error:
print(str(error))
return None
except httplib.UnknownTransferEncoding as error:
print(str(error))
return None
except httplib.UnimplementedFileMode as error:
print(str(error))
return None
except httplib.IncompleteRead as error:
print(str(error))
return None
except httplib.ImproperConnectionState as error:
print(str(error))
return None
except httplib.CannotSendRequest as error:
print(str(error))
return None
except httplib.CannotSendHeader as error:
print(str(error))
return None
except httplib.ResponseNotReady as error:
print(str(error))
return None
except httplib.BadStatusLine as error:
print(str(error))
return None
return str(rest_response)
class GoogleSuggestions():
def __init__(self):
self.hl = "en"
self.conn = None
def prepareQuery(self):
#GET /complete/search?output=toolbar&client=youtube-psuggest&xml=true&ds=yt&hl=en&jsonp=self.gotSuggestions&q=s
#self.prepQuerry = "/complete/search?output=toolbar&client=youtube&xml=true&ds=yt&"
self.prepQuerry = "/complete/search?output=chrome&client=chrome&"
if self.hl is not None:
self.prepQuerry = self.prepQuerry + "hl=" + self.hl + "&"
self.prepQuerry = self.prepQuerry + "jsonp=self.gotSuggestions&q="
print "[MyTube - GoogleSuggestions] prepareQuery:", self.prepQuerry
def getSuggestions(self, queryString):
self.prepareQuery()
if queryString is not "":
query = self.prepQuerry + quote(queryString)
self.conn = HTTPConnection("google.com")
try:
self.conn = HTTPConnection("google.com")
self.conn.request("GET", query, "",
{"Accept-Encoding": "UTF-8"})
except (CannotSendRequest, gaierror, error):
self.conn.close()
print "[MyTube - GoogleSuggestions] Can not send request for suggestions"
return None
else:
try:
response = self.conn.getresponse()
except BadStatusLine:
self.conn.close()
print "[MyTube - GoogleSuggestions] Can not get a response from google"
return None
else:
if response.status == 200:
data = response.read()
header = response.getheader(
"Content-Type", "text/xml; charset=ISO-8859-1")
charset = "ISO-8859-1"
try:
charset = header.split(";")[1].split("=")[1]
print "[MyTube - GoogleSuggestions] Got charset %s" % charset
except:
print "[MyTube - GoogleSuggestions] No charset in Header, falling back to %s" % charset
data = data.decode(charset).encode("utf-8")
self.conn.close()
return data
else:
self.conn.close()
return None
else:
return None
def request(self, host, handler, request_body, verbose=0):
from httplib import HTTPConnection
con = HTTPConnection(self.__proxy, self.__port)
con.connect()
if verbose == 1 or self.verbose == 1:
print "Server:\thttp://" + host + handler
print "Sending:\t%s" % request_body
con.request("POST", "http://" + host + handler, request_body)
f = con.getresponse()
self.verbose = verbose
#print f.read()
return (self.parse_response(f))
def do_query_specific_plaintext(self, host, http_port, method, route, payload = None, headers = {}):
conn = HTTPConnection(host, http_port, timeout = 120)
conn.connect()
if payload is not None:
conn.request(method, route, payload, headers)
else:
conn.request(method, route, "", headers)
response = conn.getresponse()
if response.status == 200:
return response.read()
else:
raise BadServerResponse(response.status, response.reason)
def http_purge_url(url):
"""
Do an HTTP PURGE of the given asset.
The URL is run through urlparse and must point
to the varnish instance not the varnishadm
"""
url = urlparse(url)
connection = HTTPConnection(url.hostname, url.port or 80)
connection.request('PURGE', '%s?%s' % (url.path or '/', url.query), '',
{'Host': url.hostname})
response = connection.getresponse()
return response
def request(method, url, data=None, headers={}, sleep=0):
scheme, netloc, path = urlsplit(urlresolve(url))
try:
conn = HTTPConnection(str(netloc))
conn.request(method, '/%s' % path, body=data, headers=headers)
response = conn.getresponse()
status = response.status
errmsg = response.reason
except httplib.HTTPException, e:
status = None
errmsg = str(e) or repr(e)
def test_badpath_redirect(webapp):
connection = HTTPConnection(webapp.server.host, webapp.server.port)
connection.connect()
path = "/../../../../../../etc/passwd"
connection.request("GET", path)
response = connection.getresponse()
assert response.status == 301
assert response.reason == "Moved Permanently"
connection.close()
class Connection(object):
def __init__(self, host):
self.connection = HTTPConnection(host)
self.host = host
def request(self, method, url, body=None, headers={}):
self.connection.request(method, url, body, headers)
response = self.connection.getresponse()
content = response.read() or None
mimetype = response.getheader('Content-Type', None)
return HttpResponse(STATUS_CODES[response.status], content, mimetype)
def _get_response(self, url):
"""Return response object from URL"""
try:
conn = HTTPConnection(url)
conn.timeout = 3
conn.request('HEAD', '/')
return conn.getresponse()
except socket.error:
return None
except Exception as e:
self.log.error(e, exc_info=True)
return None
def request(self, path, method=None, params=None, debug=False, stats=False):
"""
Sends a request and gets a response from isbndb.com
@param: method should be in (GET, POST, DELETE, or PUT)
@param: params should be a dictionary of options to send to server
@param: debug if true, reports the arguments that you sent to the server
@param: stats if true, reports the statistics of the key in use.
"""
params = params or { }
params['access_key'] = self.auth
if debug:
params['results'] = 'args'
if stats:
params['results'] = 'keystats'
params = urlencode(params)
query = None
data = None
if not path or len(path) < 1:
raise ValueError('Invalid path parameter')
if method and method not in ['GET', 'POST', 'DELETE', 'PUT']:
raise NotImplementedError('HTTP %s method not implemented' % method)
if path[0] == '/':
uri = self.base
else:
uri = self.base + path
if method == "GET":
query = params
elif method == "POST" or method == "PUT":
data = params
headers = {
"User-Agent":"ISBNdb-Python",
}
conn = HTTPConnection(self.host)
if query:
conn.request(method, '?'.join([uri, query]), '', headers)
elif data:
conn.request(method, uri, data, headers)
response = conn.getresponse( )
if response.status != 200:
raise ISBNdbHttpException(response.status, uri, response.reason)
else:
return parse(response)
class BlobWriter(object):
def __init__(self,
length,
model=None,
id=None,
name=None,
group=None,
filename=''):
if model:
url = '/blob/{0}/{1}/{2}/{3}'.format(urllib.quote(model), str(id),
urllib.quote(group),
urllib.quote(name))
else:
url = '/blob/upload'
headers = {}
headers['Host'] = urllib.quote(client_host)
headers['Cookie'] = 'satori_token=' + urllib.quote(
token_container.get_token())
headers['Content-length'] = str(length)
headers['Filename'] = urllib.quote(filename)
if ssl:
self.con = HTTPSConnection(client_host, blob_port)
else:
self.con = HTTPConnection(client_host, blob_port)
try:
self.con.request('PUT', url, '', headers)
except:
self.con.close()
raise
def write(self, data):
try:
ret = self.con.send(data)
except:
self.con.close()
raise
return ret
def close(self):
try:
res = self.con.getresponse()
if res.status != 200:
raise Exception("Server returned %d (%s) answer." %
(res.status, res.reason))
length = int(res.getheader('Content-length'))
ret = res.read(length)
finally:
self.con.close()
return ret
def contact_vessel(self, vessel_ip, path, action, key, value):
# the Boolean variable we will return
success = False
print "Contacting vessel " + vessel_ip
# The variables must be encoded in the URL format, through urllib.urlencode
post_content = urlencode({'action': action, 'key': key, 'value': value})
# the HTTP header must contain the type of data we are transmitting, here URL encoded
headers = {"Content-type": "application/x-www-form-urlencoded"}
# We should try to catch errors when contacting the vessel
try:
# We contact vessel:PORT_NUMBER since we all use the same port
# We can set a timeout, after which the connection fails if nothing happened
connection = HTTPConnection("%s:%d" % (vessel_ip, PORT_NUMBER), timeout = 25)
# We only use POST to send data (PUT and DELETE not supported)
action_type = "POST"
# We send the HTTP request
connection.request(action_type, path, post_content, headers)
# We retrieve the response
response = connection.getresponse()
# We want to check the status, the body should be empty
status = response.status
# If we receive a HTTP 200 - OK
if status == 200:
success = True
# We catch every possible exceptions
except Exception as e:
print "Error while contacting %s" % vessel_ip
# printing the error given by Python
print(e)
if not self.leaderElected:
self.neighborNumber = self.neighborNumber % len(self.vessels) + 1
self.nextNeighbor = "10.1.0." + str(self.neighborNumber)
print "NextNeighbor changed to " + self.nextNeighbor
print "Resending election message.."
#Resend election message
self.contact_vessel(self.nextNeighbor, '/elect', 'POST', self.addSelf(), self.sendDicts)
'''
If a node unsuccessfully contacts the leader, it will choose a new leader and inform
the other nodes
'''
if vessel_ip == self.leader:
#Choose new leader, based on the random value we store in the sortedCandidates-list
self.timesFailed = self.timesFailed + 1
self.leader = self.sendDicts[self.sortedCandidates[self.timesFailed % len(self.vessels)]]
self.propagate_value_to_vessels('/newLeader', 'POST', 0, self.leader)
print "New leader: " + self.leader
#Send the request again
self.contact_vessel(self.leader, path, action, key, value)
return success
def get_koalemos_status():
conn = HTTPConnection(OPTIONS['host'], OPTIONS['port'])
conn.request('GET', '/puzzle')
res = conn.getresponse()
if res.status != 200:
sys.stderr.write(
'''[x] get koalemos status failed: %s, %s
''' % res.reason, res.read())
sys.exit(2)
dat = res.read()
s, x, n = dat.split(',')
return s, x, int(n)
def http_size(host_name, path):
try:
from httplib import HTTPConnection
except ImportError:
from http.client import HTTPConnection
try:
conn = HTTPConnection(host_name, timeout=3)
conn.request("GET", path)
resp = conn.getresponse()
return int(resp.getheader("content-length"))
except Exception as e:
print(e)
return 0
def _getAnagram(self, text):
con = HTTPConnection("anagramgenius.com", timeout=20)
params = {"source_text" : text.encode("utf-8")}
data = None
try:
con.request("GET", "/server.php?%s" % (urlencode(params)))
data = con.getresponse().read().decode("utf-8")
except Exception as e:
self.logger.warning("Failed to retrieve a valid response from the anagram server.")
self.logger.debug(e)
finally:
con.close()
return data
def check_is_active_rm(rm_web_host, rm_web_port):
conn = HTTPConnection(rm_web_host, rm_web_port)
try:
conn.request('GET', '/cluster')
except:
return False
response = conn.getresponse()
if response.status != OK:
return False
else:
if response.getheader('Refresh', None) is not None:
return False
return True
def check_url(url):
"""
Checks if url is reachable
:param url: url
:return: True if successful, False otherwise
"""
p = urlparse(url)
conn = HTTPConnection(p.netloc)
conn.request('HEAD', p.path)
resp = conn.getresponse()
return resp.status < 400
