def fetch_bizrate_page(url_part, data=None):
h = Http()
req_url = 'http://www.bizrate.co.uk/' + url_part
if data:
return h.request(req_url, 'POST', data)
else:
return h.request(req_url, 'GET')
def get(self):
code = self.request.get("code", None)
if code is None:
self.redirect(AUTH_URL)
else:
h = Http()
data = dict(
code=code,
client_id=CLIENT_ID,
client_secret=CLIENT_SECRET,
redirect_uri=REDIRECT_URI,
grant_type="authorization_code",
)
resp, access_content = h.request("https://accounts.google.com/o/oauth2/token", "POST", urlencode(data))
# this call gets us email info. however, decrypting the ID_TOKEN would have done it too
userinfo_endpoint = "https://www.googleapis.com/oauth2/v3/userinfo"
data = json.loads(access_content)
headers = {"Authorization": "Bearer " + data["access_token"]}
resp, profile_content = h.request(userinfo_endpoint, "GET", headers=headers)
drive_endpoint = "https://www.googleapis.com/drive/v2/files"
resp, drive_content = h.request(drive_endpoint, "GET", headers=headers)
self.response.out.write(access_content + "<br><br>" + profile_content + "<br><br>")
driveFiles = json.loads(drive_content)
for file in driveFiles["items"]:
self.response.out.write('<p><img src="' + file["iconLink"] + '"> ' + file["title"] + "</p>")
def DownloadSubs(self, fileData, lang):
# Start an http object
h = Http()
# Get episode hash
epHash = self.get_hash(fileData.GetFilePath())
# Construct Url
url = SubDBUrl_Search.replace('HASH_ALIAS', epHash)
# Search for subs
resp, content = h.request(url, "GET", headers={'user-agent': MyUserAgent})
if (resp.status != 200):
return False
# Get supported languages that actually interest us.
langsFound = str(content)[2:-1].split(',')
if lang not in langsFound:
return False
# Download subtitles for all languages
# Construct Url
url = SubDBUrl_Download.replace('HASH_ALIAS', epHash)
# Replace language alias in url
url = url.replace('LANGUAGE_ALIAS', lang)
# Download subtitle
resp, content = h.request(url, "GET", headers={'user-agent': MyUserAgent})
if (resp.status != 200):
return False
# Save the subtitle file
self.SaveSubtitleFile(fileData, lang, content)
return True
def purge_version(version, mainsite=False, subdomain=False, cname=False):
varnish_servers = getattr(settings, 'VARNISH_SERVERS', None)
h = Http()
if varnish_servers:
for server in varnish_servers:
if subdomain:
#Send a request to the Server, to purge the URL of the Host.
host = "%s.readthedocs.org" % version.project.slug
headers = {'Host': host}
url = "/en/%s/*" % version.slug
to_purge = "http://%s%s" % (server, url)
print "Purging %s on %s" % (url, host)
ret = h.request(to_purge, method="PURGE", headers=headers)
if mainsite:
headers = {'Host': "readthedocs.org"}
url = "/docs/%s/en/%s/*" % (version.project.slug, version.slug)
to_purge = "http://%s%s" % (server, url)
print "Purging %s on readthedocs.org" % url
ret = h.request(to_purge, method="PURGE", headers=headers)
if cname:
redis_conn = redis.Redis(**settings.REDIS)
for cnamed in redis_conn.smembers('rtd_slug:v1:%s' % version.project.slug):
headers = {'Host': cnamed}
url = "/en/%s/*" % version.slug
to_purge = "http://%s%s" % (server, url)
print "Purging %s on %s" % (url, cnamed)
ret = h.request(to_purge, method="PURGE", headers=headers)
print ret
class Client(object):
def __init__(self, server_name):
self.http = Http()
self.server_name = 'https://' + server_name
def get(self, action_path, *values):
path = action_path + '/'.join([quote(unicode(value)) for value in values])
url = ''.join((self.server_name, path))
response, content = self.http.request(url)
if response['status'] == '500':
raise FailedApiCall(GetResponse(content).error, url)
if response['status'] != '200':
msg = 'Server `%s` answered %s status for `%s`.\n%s'
raise BadResponse(msg % (self.server_name,
response['status'],
url,
content))
if response['content-type'] == 'text/xml':
msg = 'Server `%s` answered %s content-type for `%s`.\n%s'
raise BaseResponse(msg % (self.server_name,
response['content-type'],
url,
content))
response = GetResponse(content)
return response
def post(self, path, data):
url = self.server_name + path
response, content = self.http.request(url, 'POST', data.encode('latin1', 'ignore'))
return PostResponse(content)
def list_supported_auths_and_fields():
h = Http()
resp, content = h.request(rest_url + 'authenticators/types', headers=headers)
if resp['status'] != '200':
print "Error in request: \n-------------------\n{}\n{}\n----------------".format(resp, content)
sys.exit(1)
r = json.loads(content)
for auth in r: # r is an array
print '* {}'.format(auth['name'])
for fld in auth: # every auth is converted to a dictionary in python by json.load
# Skip icon
if fld != 'icon':
print " > {}: {}".format(fld, auth[fld])
resp, content = h.request(rest_url + 'authenticators/gui/{}'.format(auth['type']), headers=headers)
if resp['status'] != '200':
print "Error in request: \n-------------------\n{}\n{}\n----------------".format(resp, content)
sys.exit(1)
print " > GUI"
rr = json.loads(content)
for field in rr:
print " - Name: {}".format(field['name'])
print " - Value: {}".format(field['value'])
print " - GUI: "
for gui in field['gui']:
print " + {}: {}".format(gui, field['gui'][gui])
print " > Simplified fields:"
for field in rr:
print " - Name: {}, Type: {}, is Required?: {}".format(field['name'], field['gui']['type'], field['gui']['required'])
def refresh_window(tot_before, tot_after, pnic_before, pnic_after,cpu_state,memory_state):
os.system("clear")
"""Print stats on screen."""
print_line(time.asctime()+" | "+cpu_state+" | "+memory_state)
#print current time #cpu state #memory
#print_line(str(int(time.time()*1000))+" | "+cpu_state[6:][:-1]+" | "+memory_state.split(" ")[3][:-1])
print_line(" NetStates:")
print_line("total bytes: sent: %-10s received: %s" \
% (bytes2human(tot_after.bytes_sent),
bytes2human(tot_after.bytes_recv))
)
print_line("total packets: sent: %-10s received: %s" \
% (tot_after.packets_sent, tot_after.packets_recv)
)
h = Http()
#data = dict(platform="UNIX",time=str(int(time.time()*1000)),type="cpu_mem",data=cpu_state[6:][:-1]+","+memory_state.split(" ")[3][:-1])
data = 'http://202.112.51.64:8000?platform=UNIX&id=1&time='+str(int(time.time()*1000))+'&type=cpu_mem&data='+cpu_state[6:][:-1]+','+memory_state.split(" ")[3][:-1]
#content = urllib2.urlopen('http://202.112.51.64:8000').read()
resp,content = h.request(data, "GET")#, urlencode(data))
resp
time.sleep(1)
#data = dict(platform="UNIX",id="1",time=str(int(time.time()*1000)),type="network",data=str(tot_after.bytes_sent)+","+str(tot_after.bytes_recv)+","+str(tot_after.packets_sent)+","+str(tot_after.packets_recv))
data = 'http://202.112.51.64:8000?platform=UNIX&id=1&time='+str(int(time.time()*1000))+'&type=network&data='+str(tot_after.bytes_sent)+","+str(tot_after.bytes_recv)+","+str(tot_after.packets_sent)+","+str(tot_after.packets_recv)
resp,content = h.request(data, "GET")#, urlencode(data))
resp
#content = urllib2.urlopen(data).read()
time.sleep(1)
class Prudence:
def __init__(self,user,pwd,host):
self.h = Http()
self.host = host
users_type = {True:Super_user,False:User}
data = dict(loginPassword=hashlib.sha1(pwd).hexdigest(),loginUsername=user)
self.headers = {'Content-type': 'application/x-www-form-urlencoded'}
resp, content = self.h.request("http://"+host+":8080/bb/data/login/","POST",headers=self.headers, body= urlencode(data))
self.raw_user = json.loads(content)['user']
self.User = users_type[self.raw_user['superuser']](self.raw_user)
self.contacts = []
def get_user(self):
return self.User
def get_raw_user(self):
return self.raw_user
def get_contacts(self):
if len(self.contacts)==0:# meccanismo di caching
data = dict(session_id=self.User.get_session_id())
r, c = self.h.request("http://"+self.host+":8080/bb/data/contacts/"+self.User.get_parameters(),"GET",headers=self.headers, body= urlencode(data))
self.contacts = json.loads(c).get('data')
return self.contacts
def get_companies(self):
data = dict(session_id=self.User.get_session_id())
r, c = self.h.request("http://"+self.host+":8080/bb/data/companies/"+self.User.get_parameters(),"GET",headers=self.headers, body= urlencode(data))
self.companies = json.loads(c).get('data')
return self.companies
def submit(self, method, request_uri, content=None, content_type=None,
content_length=None, chunked=False):
headers = { 'Connection' : 'Keep-Alive'
, 'Keep-Alive' : '300'
}
repository = self.getRepositoryURL()
url = self.getRequestURL(request_uri)
#
http = Http()
#http.add_credentials(self.auth_user, self.auth_pwd, self.domain)
auth = base64.encodestring("%s:%s" % (self.auth_user, self.auth_pwd))
headers['Authorization'] = 'Basic ' + auth
if content is None:
self._last_request = '%s ' % method + url
response, body = http.request(url, method, headers=headers)
else:
self._last_request = '%s (%s) ' % (method, content_type) + url
headers['Content-Type'] = content_type
if content_length:
headers['Content-Length'] = str(content_length)
response, body = http.request(url, method, body=content,
headers=headers)
response = FCRepoResponse(repository, method, request_uri,
response, body)
return response
def get(self):
self.response.headers['Content-Type'] = 'application/json'
code = self.request.get('code')
from httplib2 import Http
from urllib import urlencode
h = Http()
data = dict(redirect_uri='http://localhost:8080/githubauthback', client_id='baccad0153a85f91041f',
code=code, client_secret='aad9fc6fbd5f9cca609c76a7b693d3f14bf56bae')
headers = {'Accept': 'application/json'}
resp, data = h.request('https://github.com/login/oauth/access_token', 'POST', headers=headers, body=urlencode(data))
data = json.loads(data)
self.response.write('Git Hub :\n\n')
self.response.write(resp)
self.response.write('\n\n')
self.response.write(str(data))
headers = {'Authorization': 'token ' + data['access_token']}
resp, data = h.request('https://api.github.com/user', 'GET', headers=headers)
data = json.loads(data)
self.response.write('\n\n')
self.response.write(resp)
self.response.write('\n\n')
self.response.write(str(data))
def test_epgguide(self):
cache_dir = os.getcwd() + "/.epguide"
# os.makedirs(cache_dir)
print cache_dir
cache = FileCache(cache_dir)
h = Http(cache = cache)
user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0"
httplib2.debuglevel = 255
resp, content = h.request("http://www.teleman.pl/program-tv/stacje/TVP-2", headers={'user-agent':user_agent})
print "content:"
print content
print "response:"
print resp
print "fromcache:" + str(resp.fromcache)
print "status:" + str(resp.status)
#url = "http://www.teleman.pl/program-tv/stacje/TVP-2"
url = "http://www.teleman.pl/tv/Dr-House-7-152-885990"
resp, content = h.request(url, headers={'user-agent':user_agent})
print "fromcache:" + str(resp.fromcache)
print "status:" + str(resp.status)
safe = httplib2.safename(url)
print "safe:" + safe
cached_value = cache.get(url)
info, cached = cached_value.split('\r\n\r\n', 1)
print "===="
print content
print "===="
print cached
print "===="
self.assertEqual(content, cached)
def submit(self):
#Body content.
flattened = list(itertools.chain(*self.parts))
flattened.append('--' + self.boundary + '--')
flattened.append('')
body = str('\r\n'.join(flattened))
#Header content.
authheader = "Basic %s" % base64.encodestring('%s:%s' % (self.user, self.password))[:-1]
headerType = 'multipart/form-data; boundary=%s' % self.boundary
headers = {}
headers['Authorization'] = authheader
headers['Content-type'] = headerType
defaultTimeout = socket.getdefaulttimeout()
socket.setdefaulttimeout (30)
try:
h = Http()
response, content = h.request(self.url, "POST", body=body, headers=headers)
print ("httpPost Response " + str(response))
if response.get("status") == "401":
response, content = h.request(self.url, "POST", body=body, headers=headers)
print ("httpPost Unauthorized, retryPost response " + str(response))
# store cookie used for the furture _httpPost
#cookie = response.get('cache-control')
except urllib2.HTTPError, e:
print ('HTTP Response Code: %s')
raise e
def get_data(*args):
#TODO Pass in a timeout value here
from httplib2 import Http
from urllib import urlencode
req = Http(timeout=10)
if (Prefs['authrequired']):
req.add_credentials(Prefs['username'], Prefs['password'])
results = []
for item in args:
u = item[0]
data = item[1]
print data
try:
if data:
headers = {'Content-type': 'application/x-www-form-urlencoded'}
resp, content = req.request(u, "POST", headers=headers, body=urlencode(data))
print resp, content
else:
resp, content = req.request(u, "GET", data)
soup = BeautifulSoup(content)
results.append(soup)
except:
raise
return results
def test_parse_title_from_url(self):
with Mock() as Http:
from httplib2 import Http
connection = Http()
connection.request(self.url) >> ({'headers': ''}, self.html)
html2data_instance = HTML2Data(url = self.url)
title = html2data_instance.xpath('//head/title/text()')
self.assertEqual(['Example Page',], title)
def _mockLastFMRequest(self, body_path = 'fixtures/last_fm/vivaldi.xml', read_format = 'r'):
with Stub() as Http:
from httplib2 import Http
connection = Http()
headers_path = os.path.join(os.path.dirname(__file__), 'fixtures/last_fm/vivaldi_headers.json')
body_path = os.path.join(os.path.dirname(__file__), body_path)
connection.request(any()) >> (
json.load(open(headers_path)),
open(body_path, read_format).read()
)
def printproxy(request):
url = request.params.get("url")
if url is None:
return HTTPBadRequest()
# check for full url
parsed_url = urlparse(url)
if not parsed_url.netloc or parsed_url.scheme not in ("http", "https"):
return HTTPBadRequest()
printpath = request.params.get('path')
method = request.method
body = None
if method in ('POST', 'PUT'):
body = request.body
# forward request to target (without Host Header)
http = Http(disable_ssl_certificate_validation=True)
h = dict(request.headers)
h.pop("Host", h)
try:
if url:
resp, content = http.request(BASE_URL + str(printpath) + "?url=" + url, method=method, body=body, headers=h)
else:
resp, content = http.request(BASE_URL, method=request.method,
body=request.body, headers=h)
except:
return HTTPBadGateway()
headers = {}
if resp.has_key("content-type"):
headers["Content-Type"] = resp["content-type"]
if resp.has_key("set-cookie"):
c = Cookie.SimpleCookie()
c.load(resp["set-cookie"])
morsel = c.get('SRV')
if morsel is not None:
#morsel['max-age'] = 60 * 15 # seconds
morsel['path'] = request.path
headers["Set-Cookie"] = "SRV=%s; path=%s" % ( morsel.value, morsel['path'])
if resp.has_key("Cookie"):
headers["Cookie"] = resp["Cookie"]
if resp.has_key("Content-Disposition"):
headers["Content-Disposition"] = resp["Content-Disposition"]
if resp.has_key("content-disposition"):
headers["Content-Disposition"] = resp["content-disposition"]
response = Response(content, status=resp.status,
headers=headers)
return response
def get_tides(url, key, location):
yesterday = datetime.now() - timedelta(days=1)
h = Http()
resp, tides_json = h.request("%s/%s/tide_%s/q/%s.json" % (url, key, yesterday.strftime('%Y%m%d'), location))
tides = json.loads(tides_json)
tidelist = filter(is_tide, tides['tide']['tideSummary'])
resp, levels_json = h.request("%s/%s/rawtide/q/%s.json" % (url, key, location))
levels = json.loads(levels_json)
return (tidelist, levels)
def insertPaintingsIntoDatabase(filteredPaintings):
#clear and add paintings
apikey= "?apiKey=50c6ab59e4b05e32287d6e3d"
fixedURL= "https://api.mongolab.com/api/1/databases/art_history/collections/paintings"
url = fixedURL + apikey
h = Http()
resp, content = h.request(url, "PUT", body=simplejson.dumps([]), headers={'content-type':'application/json'} )
h = Http()
resp, content = h.request(url, "POST", body=simplejson.dumps(filteredPaintings), headers={'content-type':'application/json'} )
def _pdf(self):
body = {
"comment": "Foobar",
"title": "Bouchon",
"units": "m",
"srs": "EPSG:%i" % self.request.registry.settings["srid"],
"dpi": 254,
"layers": [],
"layout": self.settings["print_template"],
"pages": [{
"center": [self.settings["print_center_lon"], self.settings["print_center_lat"]],
"col0": "",
"rotation": 0,
"scale": self.settings["print_scale"],
"table": {
"columns": ["col0"],
"data": [{
"col0": ""
}]
}
}]
}
body = dumps(body)
_url = add_url_params(self.request.route_url("printproxy_create"), {
"url": self.request.route_url("printproxy"),
})
h = Http()
log.info("Checker for printproxy request (create): %s" % _url)
_url = _url.replace(self.request.environ.get("SERVER_NAME"), "localhost")
headers = {
"Content-Type": "application/json;charset=utf-8",
"Host": self.request.environ.get("HTTP_HOST")
}
resp, content = h.request(_url, "POST", headers=headers, body=body)
if resp.status != httplib.OK:
self.set_status(resp.status, resp.reason)
return "Failed creating PDF: " + content
log.info("Checker for printproxy pdf (retrieve): %s" % _url)
json = loads(content)
_url = json["getURL"].replace(self.request.environ.get("SERVER_NAME"), "localhost")
headers = {"Host": self.request.environ.get("HTTP_HOST")}
resp, content = h.request(_url, headers=headers)
if resp.status != httplib.OK:
self.set_status(resp.status, resp.reason)
return "Failed retrieving PDF: " + content
return "OK"
class SimpleRestClient:
"""A simple REST client library"""
def __init__(self, api_url, api_user, api_password):
self.url, self.user, self.password = api_url, api_user, api_password
self.client = Http()
self.client.follow_all_redirect = True
self.client.add_credentials(self.user, self.password)
def GET(self, uri):
if uri.startswith("http://"):
current_url = ""
else:
current_url = self.url
status, response = self.client.request(
"{url}{uri}".format(url=current_url, uri=uri), "GET", headers={"accept": "application/xml"}
)
response = self.parse_xml(response)
return status, response
def POST(self, uri, params={}):
if uri.startswith("http://"):
current_url = ""
else:
current_url = self.url
if not params:
params = {}
status, response = self.client.request(
"{url}{uri}".format(url=current_url, uri=uri),
"POST",
urlencode(params),
headers={"accept": "application/xml"},
)
response = self.parse_xml(response)
return status, response
def DELETE(self, uri):
if uri.startswith("http://"):
current_url = ""
else:
current_url = self.url
return self.client.request("{url}{uri}".format(url=current_url, uri=uri), "DELETE")
def PUT(self, uri):
if uri.startswith("http://"):
current_url = ""
else:
current_url = self.url
return self.client.request("{url}{uri}".format(url=current_url, uri=uri), "PUT")
def parse_xml(self, response):
return libxml2.parseDoc(response)
def test_httplib2_in_out():
hostname = str(uuid4())
port = 9999
url = 'http://%s:%s/' % (hostname, port)
http = Http()
with Httplib2Interceptor(app=app, url=url) as target_url:
response, content = http.request(target_url)
assert response.status == 200
assert 'WSGI intercept successful!' in content.decode('utf-8')
# outside the context manager the intercept does not work
with py.test.raises(ServerNotFoundError):
http.request(url)
def _pdf(self):
body = {
'comment': 'Foobar',
'title': 'Bouchon',
'units': 'm',
'srs': "EPSG:%i" % self.request.registry.settings['srid'],
'dpi': 254,
'layers': [],
'layout': self.settings['print_template'],
'pages': [{
'center': [self.settings['print_center_lon'], self.settings['print_center_lat']],
'col0': '',
'rotation': 0,
'scale': self.settings['print_scale'],
'table': {
'columns': ["col0"],
'data': [{
'col0': ''
}]
}
}]
}
body = dumps(body)
_url = self.request.route_url('printproxy_create') + \
'?url=' + self.request.route_url('printproxy')
h = Http()
log.info("Checker for printproxy request (create): %s" % _url)
_url = _url.replace(self.request.environ.get('SERVER_NAME'), "localhost")
headers = {
'Content-Type': 'application/json;charset=utf-8',
'Host': self.request.environ.get('HTTP_HOST')
}
resp, content = h.request(_url, 'POST', headers=headers, body=body)
if resp.status != httplib.OK:
self.set_status(resp.status, resp.reason)
return 'Failed creating PDF: ' + content
log.info("Checker for printproxy pdf (retrieve): %s" % _url)
json = loads(content)
_url = json['getURL'].replace(self.request.environ.get('SERVER_NAME'), "localhost")
headers = {'Host': self.request.environ.get('HTTP_HOST')}
resp, content = h.request(_url, headers=headers)
if resp.status != httplib.OK:
self.set_status(resp.status, resp.reason)
return 'Failed retrieving PDF: ' + content
return 'OK'
def send_request_to_google_analytics(utm_url):
# Make a tracking request to Google Analytics from this server.
# Copies the headers from the original request to the new one.
http = Http()
try:
http.request(utm_url, "GET", headers = {
'User-Agent': ENVIRON.get('HTTP_USER_AGENT', 'Unknown'),
'Accepts-Language:': ENVIRON.get('HTTP_ACCEPT_LANGUAGE','')
})
dbgMsg("success")
except HttpLib2Error:
raise Exception("Error opening: %s" % utm_url)
def _pdf(self):
body = {
"comment": "Foobar",
"title": "Bouchon",
"units": "m",
"srs": "EPSG:%i" % self.request.registry.settings["srid"],
"dpi": 254,
"layers": [],
"layout": self.settings["print_template"],
"pages": [{
"center": [self.settings["print_center_lon"], self.settings["print_center_lat"]],
"col0": "",
"rotation": 0,
"scale": self.settings["print_scale"],
"table": {
"columns": ["col0"],
"data": [{
"col0": ""
}]
}
}]
}
body = dumps(body)
url = add_url_params(self.request.route_url("printproxy_create"), {
"url": self.request.route_url("printproxy"),
})
url, headers = build_url("Check the printproxy request (create)", url, self.request, {
"Content-Type": "application/json;charset=utf-8",
})
h = Http()
resp, content = h.request(url, "POST", headers=headers, body=body)
if resp.status != httplib.OK:
self.set_status(resp.status, resp.reason)
return "Failed creating PDF: " + content
json = loads(content)
url, headers = build_url(
"Check the printproxy pdf (retrieve)", json["getURL"], self.request
)
resp, content = h.request(url, headers=headers)
if resp.status != httplib.OK:
self.set_status(resp.status, resp.reason)
return "Failed retrieving PDF: " + content
return "OK"
def hostmeta_for_domain(domain):
# Try to get the host-meta document.
http = Http()
try:
response, content = http.request('https://%s/.well-known/host-meta?format=json' % domain)
except socket.error, exc:
if exc.errno != errno.ECONNREFUSED:
raise
try:
response, content = http.request('http://%s/.well-known/host-meta?format=json' % domain)
except socket.error, exc:
if exc.errno == errno.ECONNREFUSED:
raise BadResponse('Connection refused')
raise
def request(self, path, options = {}):
client = Http()
if len(options) == 0:
resp, content = client.request(path)
else:
resp, content = client.request(
path,
options['request_type'],
body = options['body'],
headers = options['headers']
)
return loads(content)
class SocrataBase:
"""Base class for all Socrata API objects"""
def __init__(self, configuration):
"""
Initializes a new Socrata API object with configuration
options specified in standard ConfigParser format
"""
self.config = configuration
self.username, password, host, api_host = (self.config.get('credentials', 'user'),
self.config.get('credentials', 'password'),
self.config.get('server', 'host'),
self.config.get('server', 'api_host'))
self.app_token = self.config.get('credentials', 'app_token')
self.api = Http()
self.url = host
self.id_pattern = re.compile('^[0-9a-z]{4}-[0-9a-z]{4}$')
response, content = self.api.request('%s/authenticate' % api_host, 'POST',
headers={'Content-type': 'application/x-www-form-urlencoded',
'X-App-Token': self.app_token},
body=urlencode({'username': self.username, 'password': password}))
cookies = re.search('(_blist_session_id=[^;]+)', response['set-cookie'])
self.cookie = cookies.group(0)
# For multipart upload/streaming
register_openers()
def _request(self, service, type, data = {}):
"""Generic HTTP request, encoding data as JSON and decoding the response"""
response, content = self.api.request(
self.url + service, type,
headers = { 'Content-type:': 'application/json',
'X-App-Token': self.app_token,
'Cookie': self.cookie },
body = json.dumps(data) )
if content != None and len(content) > 0:
response_parsed = json.loads(content)
if hasattr(response_parsed, 'has_key') and \
response_parsed.has_key('error') and response_parsed['error'] == True:
print "Error: %s" % response_parsed['message']
return response_parsed
return None
def _batch(self, data):
payload = {'requests': data}
return self._request('/batches', 'POST', payload)
def GetDocument(query=None,url=None,fields=None,filter=None,sort=None):
data = {}
flag = 0
if fields is not None:
data['fields'] = fields
flag =1
if query is not None:
data['query'] = query
if filter is not None:
data['filter'] = filter
if sort is not None:
data['sort'] = sort
h = Http()
data = jsonpickle.encode(data,unpicklable=True)
print data
resp,content = h.request(url,"GET",data)
print content
print resp
if resp.status == 200:
if flag == 0:
content = jsonpickle.decode(content)
result = []
for res in content['hits']['hits']:
result.append(res['_source'])
return result
else:
return content
else:
return resp
class GoogleType:
def __init__(self):
self.h = Http()
self._URI = "http://www.google-type.com"
self._API_URI = "%s/genimg.php" % self._URI
self._HEADERS_MUST = {"Content-Type":"application/x-www-form-urlencoded; charset=UTF-8"}
def get(self, s):
header, body = self.h.request(uri=self._API_URI, method="POST",
body=urlencode({"q":s}), headers=self._HEADERS_MUST)
if header["status"] != "200":
raise Exception("Cant Connect to %s" % self._API_URI)
return body
def parse(self, body):
p = GoogleTypeParser(body)
#return linesep.join([self._URI+s for s in p.imgSrcList])
return [self._URI + "/" + s for s in p.imgSrcList]
def printD(self, body):
p = GoogleTypeParser(body)
#return linesep.join([self._URI+s for s in p.imgSrcList])
for key in p.imgNameDict:
for s in p.imgNameDict[key]:
print "%s %s" % (key, self._URI + "/" + s)
def close_job(self, job_id):
doc = self.create_close_job_doc()
http = Http()
url = self.endpoint + "/job/%s" % job_id
resp, content = http.request(url, "POST", headers=self.headers(),
body=doc)
self.check_status(resp, content)
'Cache-Control':
'max-age=0',
'Connection':
'keep-alive',
'Host':
'www.renren.com',
'Referer':
'http://www.renren.com/Home.do',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.24',
}
headers = headers_template.copy()
headers['Content-type'] = 'application/x-www-form-urlencoded'
resp, content = h.request(LOGIN_URL,
'POST',
headers=headers,
body=urllib.urlencode(login_data))
if resp['status'] == '302':
headers = headers_template.copy()
headers['Cookie'] = resp['set-cookie']
resp, content = h.request(resp['location'], headers=headers)
headers = headers_template.copy()
headers['Host'] = 'friend.renren.com'
headers['Cookie'] = resp['set-cookie']
resp, content = h.request(FRIENDS_URL, headers=headers)
print content
class Cluster:
def __init__(self):
self.parameter_dictionary = {
"application_name": None,
"kubeapi_host_and_port": None,
"namespace": None,
"size": None,
"service_file_name": None,
"replication_controller_file_name": None,
"timeout_in_second": None,
"action": None,
}
if self.__handle_input() is False:
sys.exit(-1)
else:
print "Parameters are: " + str(self.parameter_dictionary)
self.http = Http()
self.application_name = self.parameter_dictionary.get(
"application_name", "name") #name
self.kubeapi_host_and_port = self.parameter_dictionary.get(
"kubeapi_host_and_port",
"http://127.0.0.1:8080") #"http://127.0.0.1:8080
self.size = int(self.parameter_dictionary.get("size", 1)) #3
self.service_file_name = self.parameter_dictionary.get(
"service_file_name", "service.json") #"service.json"
self.replication_controller_file_name = self.parameter_dictionary.get(
"replication_controller_file_name",
"replication-controller.json") #"replication-controller.json"
self.time_to_wait = int(
self.parameter_dictionary.get("timeout_in_second", 60)) #60 * 3
self.namespace = self.parameter_dictionary.get("namespace",
"default") #default
self.action = self.parameter_dictionary.get("action",
"create") #create
self.replication_controller_dictionary = self.__load_replication_controller_file(
)
self.service_dictionary = self.__load_service_file()
self.service_name = self.application_name #self.service_dictionary.get("metadata").get("name")
self.replication_controller_name = self.application_name #self.replication_controller_dictionary.get("metadata").get("name")
self.pod_keyword_prefix = self.replication_controller_dictionary.get(
"metadata").get("name")
def __handle_input(self):
parameter_list = sys.argv[1:]
for parameter in parameter_list:
key_value = parameter[2:]
key_value_list = key_value.split("=")
self.parameter_dictionary[key_value_list[0]] = key_value_list[1]
result = True
for key, value in self.parameter_dictionary.iteritems():
if value is None:
print "Parameter " + key + " is missing"
result = False
return result
def __load_replication_controller_file(self):
with open(self.replication_controller_file_name, "r") as file_read:
text = file_read.read()
return json.loads(text)
def __load_service_file(self):
with open(self.service_file_name, "r") as file_read:
text = file_read.read()
return json.loads(text)
# Check seed node
def __check_seed_node_up(self, pod_list):
for pod in pod_list:
head, body = self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/pods/" + pod + "/log", "GET")
for seed_node_up_keyword in self.seed_node_up_keyword_list:
if seed_node_up_keyword in body:
pass
else:
return None
return True
# Check all nodes are up in the cluster
def __check_all_node_up(self, pod_list):
for pod in pod_list:
head, body = self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/pods/" + pod + "/log", "GET")
for all_node_up_keyword in self.all_node_up_keyword_list:
if all_node_up_keyword in body:
pass
else:
return None
for all_node_failure_keyword in self.all_node_failure_keyword_list:
if all_node_failure_keyword in body:
print body
return False
return True
# Check service is up
def __check_service_up(self):
head, body = self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/services/" + self.service_name, "GET")
if head.status == 200:
return True
else:
return None
# Get all node name
def __get_all_pod_name(self, expected_size=-1):
pod_list = []
head, body = self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/pods/", "GET")
dictionary = json.loads(body)
for item in dictionary.get("items"):
if item.get("metadata").get("name").startswith(
self.pod_keyword_prefix):
pod_list.append(item.get("metadata").get("name"))
if expected_size == -1 or expected_size == len(pod_list):
return pod_list
else:
return None
def create_cluster(self):
# Create a replication controller
self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/replicationcontrollers", "POST",
json.dumps(self.replication_controller_dictionary))
# Get seed pod name
seed_pod_list = Utility.execute_until_timeout(self.__get_all_pod_name,
self.time_to_wait, 1)
if seed_pod_list is False:
print("Fail to get seed pod name")
return False
else:
print("The created seed pod is: " + str(seed_pod_list))
# Check seed pod
if Utility.execute_until_timeout(self.__check_seed_node_up,
self.time_to_wait, seed_pod_list):
print("Successfully create the seed node")
else:
print("Seed node fail to come up")
return False
# Create a service to track all nodes
self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/services", "POST",
json.dumps(self.service_dictionary))
# Check service
if Utility.execute_until_timeout(self.__check_service_up,
self.time_to_wait):
print("Successfully create the service")
else:
print("Service fail to come up")
return False
# Resize
if self.size > 1:
for i in xrange(1, self.size):
if self.resize_cluster(i + 1) is False:
return False
return True
else:
return True
def clean_cluster(self):
self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/services/" + self.service_name, "DELETE")
self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/replicationcontrollers/" +
self.replication_controller_name, "DELETE")
pod_list = self.__get_all_pod_name()
for pod in pod_list:
self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/pods/" + pod, "DELETE")
def resize_cluster(self, size):
# Get seed pod name
seed_pod_list = Utility.execute_until_timeout(self.__get_all_pod_name,
self.time_to_wait,
size - 1)
if seed_pod_list is False:
print("Fail to get seed pod name")
return False
else:
print("The created seed pod is: " + str(seed_pod_list))
# Resize
head, body = self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/replicationcontrollers/" +
self.replication_controller_name, "GET")
if head.status != 200:
print "Fail to get the current replication controller data"
print head
print body
return False
replication_controller_dictionary = json.loads(body)
replication_controller_dictionary["spec"]["replicas"] = size
head, body = self.http.request(
self.kubeapi_host_and_port + "/api/v1/namespaces/" +
self.namespace + "/replicationcontrollers/" +
self.replication_controller_name, "PUT",
json.dumps(replication_controller_dictionary))
if head.status != 200:
print "Fail to put the new replication controller data"
print head
print body
return False
# Get all pod name
pod_list = Utility.execute_until_timeout(self.__get_all_pod_name,
self.time_to_wait, size)
pod_without_seed_list = []
for pod in pod_list:
if pod in seed_pod_list:
pass
else:
pod_without_seed_list.append(pod)
print("The created pods are: " + str(pod_without_seed_list))
print("Waiting for internal synchronization")
# Check for all nodes
if Utility.execute_until_timeout(self.__check_all_node_up,
self.time_to_wait,
pod_without_seed_list):
print("Successfully resize the cluster")
return True
else:
print("One or more nodes fail to join the cluster")
return False
class Client(object):
"""
A class that has all the logic for communicating with Trello and returning
information to the user
"""
def __init__(self, api_key, user_auth_token=None):
"""
Takes the API key and User Auth Token, which are needed for all Trello
API calls to allow access to requested information
"""
self.api_key = api_key
self.user_auth_token = user_auth_token
self.client = Http()
def add_authorisation(self, query_params):
"""
Adds the API key and user auth token to the query parameters
"""
query_params['key'] = self.api_key
if self.user_auth_token:
query_params['token'] = self.user_auth_token
return query_params
def clean_path(self, path):
"""
Ensure the path has a preceding /
"""
if path[0] != '/':
path = '/' + path
return path
def check_errors(self, uri, response):
"""
Check HTTP reponse for known errors
"""
if response.status == 401:
raise Unauthorised(uri, response)
if response.status != 200:
raise ResourceUnavailable(uri, response)
def build_uri(self, path, query_params):
"""
Build the URI for the API call.
"""
url = 'https://api.trello.com/1' + self.cleanPath(path)
url += '?' + urlencode(query_params)
return url
def fetch_json(self,
uri_path,
http_method='GET',
query_params=None,
body=None,
headers=None):
"""
Make a call to Trello API and capture JSON response. Raises an error
when it fails.
"""
query_params = query_params or {}
headers = headers or {}
query_params = self.add_authorisation(query_params)
uri = self.build_uri(uri_path, query_params)
if http_method in ("POST", "PUT",
"DELETE") and 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
response, content = self.client.request(uri=uri,
method=http_method,
body=body,
headers=headers)
self.check_errors(uri, response)
return json.loads(content.decode('utf-8'))
def create_organisation(self, organisation_json):
"""
Create an Organisation object from a JSON object
"""
return Organisation(trello_client=self,
organisation_id=organisation_json['id'],
name=organisation_json['name'])
def create_board(self, board_json):
"""
Create Board object from a JSON object
"""
return Board(trello_client=self,
board_id=board_json['id'],
name=board_json['name'])
def create_list(self, list_json):
"""
Create List object from JSON object
"""
return List(trello_client=self,
list_id=list_json['id'],
name=list_json['name'])
def create_card(self, card_json):
"""
Create a Card object from JSON object
"""
return Card(trello_client=self,
card_id=card_json['id'],
name=card_json['name'])
def create_checklist(self, checklist_json):
"""
Create a Checklist object from JSON object
"""
return Checklist(trello_client=self,
checklist_id=checklist_json['id'],
name=checklist_json['name'])
def create_member(self, member_json):
"""
Create a Member object from JSON object
"""
return Member(trello_client=self,
member_id=member_json['id'],
name=member_json['fullName'])
# Deprecated method names
def addAuthorisation(self, query_params):
return self.add_authorisation(query_params)
def cleanPath(self, path):
return self.clean_path(path)
def checkErrors(self, uri, response):
self.check_errors(uri, response)
def buildUri(self, path, query_params):
return self.build_uri(path, query_params)
def fetchJson(self,
uri_path,
http_method='GET',
query_params={},
body=None,
headers={}):
return self.fetch_json(uri_path, http_method, query_params, body,
headers)
def createOrganisation(self, organisation_json):
return self.create_organisation(organisation_json)
def createBoard(self, board_json):
return self.create_board(board_json)
def createList(self, list_json):
return self.create_list(list_json)
def createCard(self, card_json):
return self.create_card(card_json)
def createChecklist(self, checklist_json):
return self.create_checklist(checklist_json)
def createMember(self, member_json):
return self.create_member(member_json)
def worker(wrk_num):
# Initialize a zeromq context
context = zmq.Context()
# Set up a channel to receive work from the ventilator
work_receiver = context.socket(zmq.PULL)
work_receiver.connect(ZMQ_ARRIVA)
# Set up a channel to send result of work to the results reporter
results_sender = context.socket(zmq.PUSH)
results_sender.connect("tcp://127.0.0.1:5558")
# Set up a channel to receive control messages over
control_receiver = context.socket(zmq.SUB)
control_receiver.connect("tcp://127.0.0.1:5559")
control_receiver.setsockopt(zmq.SUBSCRIBE, "")
# Set up a poller to multiplex the work receiver and control receiver channels
poller = zmq.Poller()
poller.register(work_receiver, zmq.POLLIN)
poller.register(control_receiver, zmq.POLLIN)
# Setup HTTP downloader
http = Http(timeout=10)
headers = {'Connection': 'close'}
# Loop and accept messages from both channels, acting accordingly
while True:
socks = dict(poller.poll())
# If the message came from work_receiver channel, square the number
# and send the answer to the results reporter
if socks.get(work_receiver) == zmq.POLLIN:
timingpointcode = work_receiver.recv()
sys.stdout.write('.')
sys.stdout.flush()
try:
request_timestamp = time.strftime("%Y-%m-%dT%H:%M:%S+02:00",
time.gmtime())
url = ARRIVA_REALTIME_URL % timingpointcode
response, content = http.request(url, 'GET', headers=headers)
except Exception, e:
results_sender.send('FAIL,' + timingpointcode)
send_timestamp = _parse_http_datetime(response['date'])
parsed = _parse_arriva_content(request_timestamp, send_timestamp,
timingpointcode, content)
if parsed is not None:
results_sender.send(timingpointcode + ',' + parsed)
sys.stdout.write('+')
sys.stdout.flush()
results_sender.send('FAIL,' + timingpointcode)
# If the message came over the control channel, shut down the worker.
elif socks.get(control_receiver) == zmq.POLLIN:
control_message = control_receiver.recv()
if control_message == "FINISHED":
print("Worker %i received FINSHED, quitting!" % wrk_num)
break
def _send_request(self, method, path, body=None, headers=None):
if TIMEOUTS_AVAILABLE:
http = Http(timeout=self.timeout)
url = self.base_url + path
self.log.debug("Starting request to '%s' (%s) with body '%s'...",
url, method,
str(body)[:10])
start_time = time.time()
try:
headers, response = http.request(url,
method=method,
body=body,
headers=headers)
except (IOError, AttributeError):
error_message = "Failed to connect to server at '%s'. Are you sure '%s' is correct? Checking it in a browser might help..."
params = (url, self.base_url)
self.log.error(error_message, *params, exc_info=True)
raise SolrError(error_message % params)
end_time = time.time()
self.log.info(
"Finished '%s' (%s) with body '%s' in %0.3f seconds.", url,
method,
str(body)[:10], end_time - start_time)
if int(headers['status']) != 200:
error_message = self._extract_error(headers, response)
self.log.error(
error_message,
extra={'data': {
'headers': headers,
'response': response
}})
raise SolrError(error_message)
return response
else:
if headers is None:
headers = {}
if self.scheme == 'http':
conn = HTTPConnection(self.host, self.port)
elif self.scheme == 'https':
conn = HTTPSConnection(self.host, self.port)
start_time = time.time()
self.log.debug(
"Starting request to '%s:%s/%s' (%s) with body '%s'...",
self.host, self.port, path, method,
str(body)[:10])
try:
conn.request(method, path, body, headers)
except IOError:
error_message = "Failed to connect to server at '%s'. Are you sure '%s' is correct? Checking it in a browser might help..."
params = (path, self.base_url)
self.log.error(error_message, *params, exc_info=True)
raise SolrError(error_message % params)
response = conn.getresponse()
end_time = time.time()
self.log.info(
"Finished '%s:%s/%s' (%s) with body '%s' in %0.3f seconds.",
self.host, self.port, path, method,
str(body)[:10], end_time - start_time)
if response.status != 200:
resp_headers = dict(response.getheaders())
resp_body = response.read()
error_message = self._extract_error(resp_headers, resp_body)
self.log.error(error_message,
extra={
'data': {
'headers': resp_headers,
'response': resp_body
}
})
raise SolrError(error_message)
return response.read()
def main(argv):
wp_url = ''
blog_page = ''
target = ''
try:
opts, args = getopt.getopt(argv, "hu:p:t:",
["url=", "page=", 'target='])
except getopt.GetoptError:
print 'wordpress_pingback_test.py -u <inputfile> -p <blog_page> -t <target>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'wordpress_pingback_test.py -u <inputfile> -p <blog_page> -t <target>'
sys.exit()
elif opt in ("-u", "--url"):
wp_url = arg
elif opt in ("-p", "--page"):
blog_page = arg
elif opt in ("-t", "--target"):
target = arg
if wp_url != '' and blog_page != '' and target != '':
h = Http()
print '******************************'
print '* Wordpress Pingback Tester *'
print '* roblest.com *'
print '******************************'
print '[!] Testing for Pingback Extensions'
data = '''<?xml version="1.0"?>
<methodCall>
<methodName>pingback.extensions.getPingbacks</methodName>
<params><param><string>%BLOG%</string></param></params>
</methodCall>'''
pingback = '''<?xml version="1.0" encoding="iso-8859-1"?>
<methodCall>
<methodName>pingback.ping</methodName>
<params>
<param><value><string>%TARGET%</string></value></param>
<param><value><string>%BLOG%</string></value></param>
</params>
</methodCall>'''
data = data.replace('%BLOG%', blog_page)
pingback = pingback.replace('%BLOG%',
blog_page).replace('%TARGET%', target)
resp, content = h.request(wp_url, "POST", data)
#print resp
if '<fault>' in content:
print "[-] Invalid Blog URL, please provide a valid one"
print content
elif '<array><data>' in content:
print '[+] Pingback Extension confirmed'
#print content
print '[!] Testing Pingback'
resp, content = h.request(wp_url, "POST", pingback)
if 'The pingback has already been registered' in content:
print "[-] The pingback has already been registered"
elif '<fault>' in content:
print content
print '***************'
print "[-] Not Vulnerable"
elif 'Keep the web talking! :-)' in content:
print content
print '***************'
print "[+] Vulnerable!"
else:
print 'wordpress_pingback_test.py -u <inputfile> -p <blog_page> -t <target>'
def oauth(provider):
"""
Authentication with providers
:param provider:
:return:
"""
# STEP 1 - Parse the auth code
code = request.data
if provider == 'google':
# STEP 2 - Exchange for a token
try:
# Upgrade the authorization code into a credentials object
oauth_flow = flow_from_clientsecrets(settings.BASE_DIR +
'/client_secrets.json',
scope='')
oauth_flow.redirect_uri = 'postmessage'
credentials = oauth_flow.step2_exchange(code)
except FlowExchangeError:
response = make_response(
dumps('Failed to upgrade the authorization code.'), 401)
response.headers['Content-Type'] = 'application/json'
return response
# Check that the access token is valid.
access_token = credentials.access_token
# prepare url
turl = 'https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=%s'
url = (turl % access_token)
# get result
h = Http()
result = loads(h.request(url, 'GET')[1])
# If there was an error in the access token info, abort.
if result.get('error') is not None:
response = make_response(dumps(result.get('error')), 500)
response.headers['Content-Type'] = 'application/json'
# Get user info
h = Http()
userinfo_url = "https://www.googleapis.com/oauth2/v1/userinfo"
params = {'access_token': credentials.access_token, 'alt': 'json'}
google_response = r_get(userinfo_url, params=params)
data = google_response.json()
# see if user exists, if it doesn't make a new one
user = get_user_by_email(email=data['email'])
if not user:
user = create_user(username=data.get('name'),
picture=data.get('picture'),
email=data.get('email'),
first_name=data.get('given_name'),
last_name=data.get('family_name'),
password=get_unique_str(8))
g.user = user
# create session
session['uid'] = user.id
session['provider'] = 'google'
return jsonify({'message': 'Success'}), 200
elif provider == 'facebook':
# get data
data = request.json.get('data')
# get access token
access_token = data['access_token']
# prepare path to app facebook data
fb_file = ''.join([BASE_DIR, '/facebook.json'])
# load data
fb_data = loads(open(fb_file, 'r').read())['facebook']
# gat app data
app_id = fb_data['app_id']
app_secret = fb_data['app_secret']
# prepare query url for access token
url = fb_data['access_token_url'] % (app_id, app_secret, access_token)
# get result
h = Http()
result = h.request(url, 'GET')[1]
# Use token to get user info from API
token = result.split(',')[0].split(':')[1].replace('"', '')
# prepare url for get user info
url = fb_data['user_info_url'] % token
# get result
h = Http()
result = h.request(url, 'GET')[1]
# load data
data = loads(result)
# get first name and last name
name = findall(r'[a-zA-Z]+', data['name'])
# prepare dictionary for save
user_data = dict()
user_data['provider'] = 'facebook'
user_data['username'] = ''.join(name)
user_data['first_name'] = name[0]
user_data['last_name'] = name[1]
user_data['email'] = data.get('email')
user_data['facebook_id'] = data.get('id')
user_data['access_token'] = token
# prepare url for get picture
url = fb_data['picture_url'] % token
# get result
h = Http()
result = h.request(url, 'GET')[1]
# load data
data = loads(result)
# add picture link to dictionary
user_data['picture'] = data['data']['url']
# get user info
user_info = get_user_by_email(user_data['email'])
# check the user exist, if not create a new one
if user_info is None:
user_info = create_user(username=user_data['username'],
password=get_unique_str(8),
first_name=user_data['first_name'],
last_name=user_data['last_name'],
email=user_data['email'],
picture=user_data['picture'])
g.user = user_info
# create session
session['uid'] = user_info.id
session['provider'] = 'facebook'
return jsonify({'message': 'Success'}), 200
else:
return jsonify({'error': 'Unknown provider'})
class Jxpg:
def __init__(self, username, password, base_url="http://jwxt.imu.edu.cn/"):
self.username = username
self.password = password
self.base_url = base_url
self.http = Http()
def _login(self):
login_url = self.base_url + "loginAction.do"
login_param = "zjh=" + self.username + "&mm=" + self.password
login_headers = {}
login_headers["Content-Type"] = "application/x-www-form-urlencoded"
response, content = self.http.request(uri=login_url, method="POST", body=login_param, headers=login_headers)
content = content.decode("gbk")
if "<title>学分制综合教务</title>" in content:
self.cookie = str(response["set-cookie"]).replace("; path=/", "")
return True, self.cookie
else:
return False, None
def _get_all_course(self):
url = "http://jwxt.imu.edu.cn/jxpgXsAction.do?oper=listWj"
headers = {}
headers["Cookie"] = self.cookie
response, content = self.http.request(uri=url, method="GET", headers=headers)
content = content.decode("gbk")
results = []
all_result = re.findall("<img name=\"[^\"]{0,200}\"", content)
for e in all_result:
e = e.replace(e[0:10], "")
e = e[1:len(e)-1]
results.append(e)
return results
def do(self):
login_flag, cookie = self._login()
if not login_flag:
ValueError("用户名或者密码错误")
results = self._get_all_course()
url1 = "http://jwxt.imu.edu.cn/jxpgXsAction.do"
url2 = "http://jwxt.imu.edu.cn/jxpgXsAction.do?oper=wjpg"
headers = {}
headers["Cookie"] = self.cookie
headers["Content-Type"] = "application/x-www-form-urlencoded"
count = 0
for r in results:
sp = r.split("#@")
body1 = "wjbm=" + sp[0] + "&bpr=" + sp[1] + "&pgnr=" + sp[5] + "&oper=wjShow&wjmc=&bprm=&pageSize=20&page=1¤tPage=1&pageNo="
body2 = "wjbm=" + sp[0] + "&bpr=" + sp[1] + "&pgnr=" + sp[5] + "&0000000004=5_1&0000000005=5_1&0000000006=5_1&0000000007=5_1&0000000008=5_1&0000000009=5_1&0000000010=5_1&0000000011=5_1&0000000012=5_1&0000000013=5_1&0000000014=5_1&0000000015=5_1&0000000016=5_1&0000000017=5_1&0000000018=5_1&0000000019=5_1&0000000020=5_1&0000000021=5_1&0000000002=5_1&0000000003=5_1&zgpj=%CD%A8%B9%FD%C9%CF%B1%BE%C3%C5%BF%CE%B3%CC%A3%AC%CE%D2%CC%E1%B8%DF%C1%CB%CC%E1%B3%F6%A1%A2%B7%D6%CE%F6%A1%A2%BD%E2%BE%F6%CF%E0%B9%D8%CE%CA%CC%E2%B5%C4%C4%DC%C1%A6%A1%A3"
response, content = self.http.request(uri=url1, method="POST", body=body1, headers=headers)
# print(response)
# print(content.decode("gbk"))
response, content = self.http.request(uri=url2, method="POST", body=body2, headers=headers)
# print(content.decode("gbk"))
content = content.decode("gbk")
if "评估成功" in content:
print("课程 " + sp[4] + " 评估成功")
count = count + 1
print("评估成功" + str(count) + "门课!")
def getTokenProfile(self, content):
h = Http(**self.settings.http_options)
result = simplejson.loads(content)
access_token = result['access_token']
return access_token, h.request('{}?{}'.format(self.profileEndpoint, urllib.urlencode({'access_token':access_token})), method="GET")
class HttpTransport(object):
def __init__(self, url, headers_factory):
self._api_url = url
self._headers_factory = headers_factory
self._supported_methods = (
"GET",
"POST",
"PUT",
"HEAD",
"DELETE",
)
self._attribute_stack = []
self._method = "GET"
self._posts = []
self._http = Http()
self._params = {}
self._url_template = '%(domain)s/%(generated_url)s'
self._stack_collapser = "/".join
self._params_template = '?%s'
self._retry_retries = None
self._retry_delay = None
self._retry_backoff = None
self._retry_cap = None
def setup_retry(self, retries, delay, backoff, cap):
self._retry_retries = retries
self._retry_delay = delay
self._retry_backoff = backoff
self._retry_cap = cap
def __call__(self, *args, **kwargs):
self._attribute_stack += [str(a) for a in args]
self._params = kwargs
headers = self._headers_factory()
if 'url' not in kwargs:
url = self.get_url()
else:
url = self.get_url(kwargs['url'])
if (self._method == "POST"
or self._method == "PUT") and 'type' not in kwargs:
headers.update({'content-type': 'application/json'})
# Not sure if this will always work, but for validate/verfiy nothing else was working:
body = json.dumps(kwargs)
elif 'type' in kwargs:
if kwargs['type'] == 'multipart/form-data':
body, new_headers = multipart_encode(kwargs['body'])
body = "".join(body)
headers.update(new_headers)
else:
body = kwargs['body']
headers.update({'content-type': kwargs['type']})
else:
body = self._generate_body() # hack
@_retry(self._retry_retries, self._retry_delay, self._retry_backoff,
self._retry_cap)
def _request():
response, data = self._http.request(url,
self._method,
body=body,
headers=headers)
self._attribute_stack = []
handler = kwargs.get('handler', _handle_response)
return handler(response, data)
return _request()
def _generate_params(self, params):
body = self._params_template % urlencode(params)
if body is None:
return ''
return body
def _generate_body(self):
if self._method == 'POST':
internal_params = self._params.copy()
if 'GET' in internal_params:
del internal_params['GET']
return self._generate_params(internal_params)[1:]
def _clear_content_type(self):
"""Clear content-type"""
if 'content-type' in self._headers:
del self._headers['content-type']
def _clear_headers(self):
"""Clear all headers"""
self._headers = {}
def get_url(self, url=None):
if url is None:
url = self._url_template % {
"domain":
self._api_url.rstrip('/'),
"generated_url":
self._stack_collapser(self._attribute_stack).lstrip('/'),
}
else:
url = self._url_template % {
'domain': self._api_url.rstrip('/'),
'generated_url': url.lstrip('/')
}
del self._params['url']
if len(self._params):
internal_params = self._params.copy()
if 'handler' in internal_params:
del internal_params['handler']
if self._method == 'POST' or self._method == "PUT":
if "GET" not in internal_params:
return url
internal_params = internal_params['GET']
url += self._generate_params(internal_params)
return url
def __getitem__(self, name):
self._attribute_stack.append(name)
return self
def __getattr__(self, name):
if name in self._supported_methods:
self._method = name
elif not name.endswith(')'):
self._attribute_stack.append(name)
return self
def run_url(http, ob, item):
header = {
"Host": ob['domain'],
"Connection": "keep-alive",
"Pragma": "no-cache",
"Cache-Control": "no-cache",
"Referer": item['refer'],
"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
# "Cookie": ob.get('cookie')
}
try:
url = item['url']
params = item['params']
method = item['method']
timeout = ob.get('webTimeout')
pattern = r'(login|sigin)'
p_para = r'("type":\s*"password")'
result = []
if not (re.search(pattern, url, re.I)
or re.search(p_para, json.dumps(params), re.I)):
pass
elif re.search(r'(.js|.css)', url, re.I):
pass
else:
url_parse = urlparse(url)
scheme = url_parse.scheme
domain = url_parse.netloc
path = url_parse.path
query = url_parse.query
source_ip = ob.get('source_ip')
if source_ip:
domain = source_ip
if query:
new_url = "%s://%s%s?%s" % (scheme, domain, path, query)
else:
new_url = "%s://%s%s" % (scheme, domain, path)
http = Http(timeout=timeout)
res, content = http.request(new_url, 'GET')
#pattern = r'.+(<input.*?(type="password"|type="text").*?){3}'
#se = re.search(pattern, content, re.S | re.I)
pattern3 = r'<input.*?(type="password".*?autocomplete="on")'
if re.search(pattern3, content, re.I):
response = getResponse(
res,
content,
keywords='<input.*?(type="password".*?autocomplete="on")')
request = getRequest(new_url, domain=ob['domain'])
detail = "登录页面密码输入存在密码自动补全风险"
result.append(
getRecord(ob, new_url, ob['level'], detail, request,
response))
return result
except Exception, e:
logger.error(
"File:check_login_autocomplete_url.py, run_url function :%s" %
(str(e)))
return result
def stage1():
'''Accept the Sonoff WebSocket connection, and configure it.'''
net_valid = False
conn_attempt = 0
if not args.no_check_ip:
conn_attempt = 0
# fuzzy-check if machine is connected to ITEAD AP
while True:
conn_attempt += 1
if hasfinalstageip():
print("Appear to have connected to the final stage IP, "\
"moving to next stage.")
return
if hassonoffip():
break
else:
if conn_attempt == 1:
print("** Now connect via WiFi to your Sonoff device.")
print("** Please change into the ITEAD WiFi",
"network (ITEAD-100001XXXX). The default password",
"is 12345678.")
print("To reset the Sonoff to defaults, press",
"the button for 7 seconds and the light will",
"start flashing rapidly.")
print("** This application should be kept running",
"and will wait until connected to the Sonoff...")
sleep(2)
print(".", end="", flush=True)
continue
http = Http(timeout=2)
print("~~ Connection attempt")
conn_attempt = 0
while True:
conn_attempt += 1
print(">> HTTP GET /10.10.7.1/device")
try:
resp, cont = http.request("http://10.10.7.1/device", "GET")
break
except socket_error as e:
print(e)
continue
dct = json.loads(cont.decode('utf-8'))
print("<< %s" % json.dumps(dct, indent=4))
data = {
"version": 4,
"ssid": args.wifi_ssid,
"password": args.wifi_password,
"serverName": args.serving_host,
"port": DEFAULT_PORT_HTTPS
}
print(">> HTTP POST /10.10.7.1/ap")
print(">> %s", json.dumps(data, indent=4))
resp, cont = http.request(
"http://10.10.7.1/ap", "POST", json.dumps(data))
dct = json.loads(cont.decode('utf-8'))
print("<< %s" % json.dumps(dct, indent=4))
print("~~ Provisioning completed")
def sendMessage(self, event):
url = self.webhook
status = event[0]
# Monta titulo e imagem do card
stat = None
if status == "0":
stat = "Problema"
image_url = self.PROBLEM_IMG
elif status == "1":
stat = "Resolvido"
image_url = self.RESOLVED_IMG
elif status == "2":
stat = "Reconhecido"
image_url = self.ACK_IMG
# Se for uma mensagem de problema ou resolucao
if status == "0" or status == "1":
time = event[1]
date = event[2]
trigger_name = event[3]
host_name = event[4]
severity = event[5]
self.event_id = event[6]
trigger_url = event[7]
self.trigger_id = event[8]
host_description = event[9]
bot_message = {
"cards": [{
"header": {
"title": "Severidade: " + severity,
"subtitle": stat,
"imageUrl": image_url,
"imageStyle": "IMAGE"
},
"sections": [{
"widgets": [{
"keyValue": {
"topLabel": "Alarme",
"content": trigger_name,
"contentMultiline": "true"
}
}, {
"keyValue": {
"topLabel": "Host",
"content": host_name + " " + host_description,
"contentMultiline": "true"
}
}, {
"keyValue": {
"topLabel": "Data/Hora",
"content": date + " - " + time
}
}, {
"keyValue": {
"topLabel": "ID do Evento",
"content": self.event_id
}
}]
}, {
"widgets": [{
"buttons": [{
"textButton": {
"text": "Ver o evento no ZABBIX",
"onClick": {
"openLink": {
"url":
self.zabbix_url +
"/tr_events.php?triggerid=" +
self.trigger_id + "&eventid=" +
self.event_id
}
}
}
}]
}]
}]
}]
}
# Se for uma mensagem de reconhecimento
elif status == "2":
time = event[1]
date = event[2]
ack_user = event[3]
ack_message = event[4]
event_status = event[5]
self.event_id = event[6]
self.trigger_id = event[7]
if event_status == "PROBLEM":
event_status = "Ativo"
elif event_status == "RESOLVED":
event_status = "Resolvido"
bot_message = {
"cards": [{
"header": {
"title": stat,
"subtitle": ack_user,
"imageUrl": image_url,
"imageStyle": "IMAGE"
},
"sections": [{
"widgets": [{
"keyValue": {
"topLabel": "Mensagem",
"content": ack_message,
"contentMultiline": "true"
}
}, {
"keyValue": {
"topLabel": "Status atual do alarme",
"content": event_status
}
}, {
"keyValue": {
"topLabel": "Data/Hora",
"content": date + " - " + time
}
}, {
"keyValue": {
"topLabel": "ID do Evento",
"content": self.event_id
}
}]
}, {
"widgets": [{
"buttons": [{
"textButton": {
"text": "Ver o evento no ZABBIX",
"onClick": {
"openLink": {
"url":
self.zabbix_url +
"/tr_events.php?triggerid=" +
self.trigger_id + "&eventid=" +
self.event_id
}
}
}
}]
}]
}]
}]
}
# verifica se ja possui thread, adicionando a thread na mensagem caso positivo
if self.trigger_id in self.evt_thread:
self.thread = self.evt_thread[self.trigger_id]
bot_message['thread'] = {"name": self.thread}
message_headers = {'Content-Type': 'application/json; charset=UTF-8'}
# faz requisicao http na API
http_obj = Http()
response = http_obj.request(
uri=url,
method='POST',
headers=message_headers,
body=dumps(bot_message),
)
# pega a thread da resposta da requisicao e armazena
self.thread = json.loads(response[1])['thread']['name']
event_thread = {self.trigger_id: self.thread}
self.writeEventThread(event_thread)
#GET AUTH CODE
client_url = address + "/clientOAuth"
print "Visit %s in your browser" % client_url
auth_code = ""
while auth_code == "":
auth_code = raw_input("Paste the One-Time Auth Code Here:")
#TEST ONE GET TOKEN
try:
h = Http()
url = address + "/oauth/google"
data = dict(auth_code=auth_code)
data = json.dumps(data)
resp, content = h.request(url,
'POST',
body=data,
headers={"Content-Type": "application/json"})
if resp['status'] != '200':
raise Exception('Received an unsuccessful status code of %s' %
resp['status'])
new_content = json.loads(content)
if not new_content['token']:
raise Exception('No Token Received!')
token = new_content['token']
except Exception as err:
print "Test 1 FAILED: Could not exchange auth code for a token"
print err.args
sys.exit()
else:
print "Test 1 PASS: Succesfully obtained token! "
class Service(Object):
name = 'de.taschenorakel.webradio'
interface = '%s.Service' % name
def __init__(self, bus):
def player_message_cb(bus, message):
if gst.MESSAGE_EOS == message.type:
self.__player.set_state(gst.STATE_NULL)
return True
if gst.MESSAGE_ERROR == message.type:
print message.structure and message.structure.to_string() or ''
self.__player.set_state(gst.STATE_NULL)
return True
if gst.MESSAGE_STATE_CHANGED == message.type:
if message.src == self.__player:
self.StateChanged(*self.GetState())
self.__favorites.set_state(*self.GetState())
return True
if gst.MESSAGE_TAG == message.type:
valid_types = float, int, str, unicode
tags = [(k, v) for k, v in dict(message.structure).items()
if isinstance(v, valid_types)]
self.StreamTagsChanged(dict(tags))
return True
return True
self.__data_stage = 0
self.__player = Player()
self.__player.get_bus().add_watch(player_message_cb)
self.__httplib = Http(cache=get_cache_filename())
self.__favorites = Favorites()
self.__stations = list()
self.__stream_tags = dict()
proxy = SessionBus().get_object('org.freedesktop.Notifications',
'/org/freedesktop/Notifications')
self.__notifications = Interface(proxy,
'org.freedesktop.Notifications')
self.__notify_id = 0
name = BusName(Service.name, bus)
super(Service, self).__init__(bus, '/', name)
self.__loop = MainLoop(None, True)
Thread(target=self.__load).start()
def __fetch_from_cache(self, uri):
print 'fetching from cache %s' % uri
return self.__httplib.request(
uri, headers={'cache-control': 'only-if-cached'})
def __fetch_from_web(self, uri):
print 'fetching from web %s' % uri
return self.__httplib.request(uri)
def __fetch(self, uri):
response, content = self.__fetch_from_cache(uri)
if 504 == response.status:
response, content = self.__fetch_from_web(uri)
return response, content
def __notify(self,
summary,
body=None,
id=0,
icon='rhythmbox',
app_name='webradio',
actions=None,
hints=None,
timeout=-1):
return self.__notifications.Notify(app_name or '', int(id), icon or '',
summary, body or '', actions or [],
hints or {}, int(timeout))
def __load(self):
def load_station_details(station):
response, content = self.__fetch(station.uri)
if 200 == response.status:
pattern = re.compile(r'href="([^"]+\.pls)"')
for uri in pattern.findall(content):
if uri.startswith('/'):
uri = urljoin(station.uri, uri)
if station.accept_stream(uri):
pending_channels.append([station, uri])
print '%d stations found...' % len(pending_channels)
else:
print 'Bad response: %s %s' % (response.reason,
response.status)
def find_config_file(basename):
filename = get_config_filename(basename)
if os.path.isfile(filename):
return filename
for libdir in sys.path:
prefix = os.path.commonprefix([__file__, libdir])
if not prefix or prefix != libdir:
continue
libdir_parent, libdir_name = os.path.split(libdir)
if 'site-packages' == libdir_name:
prefix = os.path.join(libdir_parent, '..', '..')
filename = os.path.join(prefix, 'share', 'webradio',
basename)
if os.path.isfile(filename):
return filename
for filename in [
os.path.join(libdir, 'data', basename),
os.path.join(libdir_parent, 'data', basename)
]:
if os.path.isfile(filename):
return filename
return None
def load_station_list():
filename = find_config_file('stations')
if filename is None:
raise RuntimeError, 'Cannot find station list'
print 'reading stations from %r' % filename
parser = SafeConfigParser()
parser.read(filename)
for station_id in parser.sections():
uri = parser.get(station_id, 'uri')
title = parser.get(station_id, 'title')
stream_uri = parser.get(station_id, 'streams')
station = Station(station_id, title, uri)
if stream_uri:
station.stream_uri = stream_uri
i = 1
while True:
key = 'noise%d' % i
i += 1
if not parser.has_option(station_id, key):
break
noise = parser.get(station_id, key)
station.add_noise_filter(noise)
for key in parser.options(station_id):
if key.startswith('alias.'):
name = key[len('alias.'):]
value = parser.get(station_id, key)
station.add_alias(name, value)
continue
load_station_details(station)
self.StationAdded(station)
def load_streams(channel, content):
parser = SafeConfigParser()
parser.readfp(StringIO(content))
playlist = dict(parser.items('playlist'))
length = int(playlist['numberofentries'])
for i in range(1, length + 1):
uri = playlist['file%d' % i]
title = playlist.get('title%d' % i)
length = int(playlist.get('length%d' % i, -1))
if title:
title = channel.station.filter_noise(title)
stream = Stream(uri, title, length)
channel.streams.append(stream)
def load_pending_channels():
for station, uri in pending_channels:
response, content = self.__fetch(uri)
if 200 == response.status:
channel = Channel(station, uri)
station.channels.append(channel)
load_streams(channel, content)
self.ChannelAdded(station.id, channel)
pending_channels = []
load_station_list()
idle_add(self.DataReady, 1)
load_pending_channels()
idle_add(self.DataReady, 2)
def run(self):
try:
self.__loop.run()
except KeyboardInterrupt:
self.__loop.quit()
@method(dbus_interface=interface,
utf8_strings=True,
in_signature='',
out_signature='a' + Station.dbus_signature)
def GetStations(self):
return self.__stations
@method(dbus_interface=interface,
utf8_strings=True,
in_signature='',
out_signature='a{sv}')
def GetStreamTags(self):
return self.__stream_tags
@method(dbus_interface=interface,
utf8_strings=True,
in_signature='as',
out_signature='a(s' + Channel.dbus_signature + ')')
def Find(self, query):
result = list()
for station in self.__stations:
for channel in station.channels:
if channel.matches(query):
match = station.id, channel
result.append(match)
return result
@method(dbus_interface=interface, in_signature='s', out_signature='')
def Play(self, uri):
self.__player.set_state(gst.STATE_NULL)
self.__player.uri = uri
self.__player.set_state(gst.STATE_PLAYING)
@method(dbus_interface=interface, in_signature='', out_signature='')
def Pause(self):
self.__player.set_state(gst.STATE_PAUSED)
@method(dbus_interface=interface, in_signature='', out_signature='')
def Resume(self):
self.__player.set_state(gst.STATE_PLAYING)
@method(dbus_interface=interface, in_signature='', out_signature='')
def Quit(self):
self.__loop.quit()
@method(dbus_interface=interface, in_signature='', out_signature='i')
def GetDataStage(self):
return self.__data_stage
@method(dbus_interface=interface, in_signature='', out_signature='bs')
def GetState(self):
playing = (gst.STATE_PLAYING == self.__player.get_state()[1])
channel_uri = self.__player.uri or ''
return playing, channel_uri
@signal(dbus_interface=interface, signature='i')
def DataReady(self, stage):
self.__data_stage = stage
@signal(dbus_interface=interface, signature=Station.dbus_signature)
def StationAdded(self, station):
self.__stations.append(station)
@signal(dbus_interface=interface, signature='s' + Channel.dbus_signature)
def ChannelAdded(self, station_id, channel):
pass
@signal(dbus_interface=interface, signature='bs')
def StateChanged(self, playing, stream_uri):
pass
@signal(dbus_interface=interface, signature='a{sv}')
def StreamTagsChanged(self, tags):
self.__stream_tags.update(tags)
summary = self.__stream_tags.get('title') or ''
body = self.__stream_tags.get('organization') or ''
self.__notify_id = self.__notify(summary, body, id=self.__notify_id)
@method(dbus_interface=interface, in_signature='', out_signature='as')
def GetTags(self):
tags = dict()
for s in self.__stations:
for c in s.channels:
for t in c.tags:
tags[t] = True
tags[s.id] = True
tags = list(tags)
tags.sort()
return tags
@method(dbus_interface=interface, in_signature='', out_signature='as')
def ListEqualizerProfiles(self):
return self.__player.get_profile_names()
@method(dbus_interface=interface, in_signature='', out_signature='s')
def GetEqualizerProfile(self):
return self.__player.profile
@method(dbus_interface=interface, in_signature='s', out_signature='')
def SetEqualizerProfile(self, profile_name):
self.__player.profile = profile_name
def _http_check(self, instance):
addr, username, password, timeout, headers, response_time, \
disable_ssl_validation, use_keystone, keystone_config, \
instance_name = self._load_http_conf(instance)
dimensions = self._set_dimensions({'url': addr}, instance)
start = time.time()
done = False
retry = False
while not done or retry:
if use_keystone:
ksclient = self._ksclients[instance_name]
token = ksclient.get_token()
if token:
headers["X-Auth-Token"] = token
headers["Content-type"] = "application/json"
else:
error_string = """Unable to get token. Keystone API server may be down.
Skipping check for {0}""".format(addr)
self.log.warning(error_string)
return False, error_string
try:
self.log.debug("Connecting to %s" % addr)
if disable_ssl_validation:
self.warning(
"Skipping SSL certificate validation for %s based on configuration" % addr)
h = Http(timeout=timeout, disable_ssl_certificate_validation=disable_ssl_validation)
if username is not None and password is not None:
h.add_credentials(username, password)
resp, content = h.request(addr, "GET", headers=headers)
except (socket.timeout, HttpLib2Error, socket.error) as e:
length = int((time.time() - start) * 1000)
error_string = '{0} is DOWN, error: {1}. Connection failed after {2} ms'.format(addr, str(e), length)
self.log.info(error_string)
return False, error_string
except httplib.ResponseNotReady as e:
length = int((time.time() - start) * 1000)
error_string = '{0} is DOWN, error: {1}. Network is not routable after {2} ms'.format(addr, repr(e), length)
self.log.info(error_string)
return False, error_string
except Exception as e:
length = int((time.time() - start) * 1000)
error_string = '{0} is DOWN, error: {1}. Connection failed after {2} ms'.format(addr, str(e), length)
self.log.error('Unhandled exception {0}. Connection failed after {1} ms'.format(str(e), length))
return False, error_string
if response_time:
# Stop the timer as early as possible
running_time = time.time() - start
self.gauge('http_response_time', running_time, dimensions=dimensions)
if int(resp.status) >= 400:
if use_keystone and int(resp.status) == 401:
if retry:
error_string = '{0} is DOWN, unable to get a valid token to connect with'.format(addr)
self.log.error(error_string)
return False, error_string
else:
# Get a new token and retry
self.log.info("Token expired, getting new token and retrying...")
retry = True
ksclient.refresh_token()
continue
else:
error_string = '{0} is DOWN, error code: {1}'.format(addr, str(resp.status))
self.log.info(error_string)
return False, error_string
done = True
return True, content
class Jasper(object):
def __init__(self,
host='localhost',
port=8080,
user='******',
pwd='jasperadmin'):
"""
Initialise new Jasper Object
"""
self.cnx = Http()
self.host = host
self.port = port
self.user = user
self.pwd = pwd
self.repo = 'jasperserver/services/repository'
self.uri = 'http://%s:%s/%s' % (self.host, self.port, self.repo)
self.headers = {
'Content-type': 'text/xml',
'charset': 'UTF-8',
'SOAPAction': 'runReport'
}
self.body = ''
def auth(self, ):
"""
Add credential
"""
self.cnx.add_credentials(self.user, self.pwd)
# We must simulate a request if we want to check the auth is correct
# Generate a soap query to verify the authentification
rq = Request(operationName='list', locale='fr_FR')
rq.append(RequestRD('folder', '', '/'))
self.body = SoapEnv('list', etree.tostring(rq)).output()
try:
res, content = self.cnx.request(self.uri, 'POST', self.body,
self.headers)
except socket.error:
raise ServerNotFound('Server not found')
if res.get('status', '200') == '401':
raise AuthError('Authentification Failed !')
elif res.get('status', '200') != '200':
return False
return True
def send(self, soapQuery=''):
try:
res, content = self.cnx.request(self.uri, 'POST', soapQuery,
self.headers)
except socket.error:
raise ServerNotFound('Server not found')
if res.get('status', '200') == '401':
raise AuthError('Authentification Failed !')
elif res.get('status', '200') != '200':
return False
try:
return self.parseMultipart(content)
except NotMultipartContent:
fp = StringIO(content)
tree = etree.parse(fp)
fp.close()
r = tree.xpath('//runReportReturn')
if not r:
raise UnknownResponse(content)
fp = StringIO(r[0].text.encode('utf-8'))
tree = etree.parse(fp)
fp.close()
raise ServerError(
'[' +
tree.xpath('//returnCode')[0].text.encode('utf-8') # noqa
+ ']' +
tree.xpath('//returnMessage')[0].text.encode('utf-8')) # noqa
def create_request(self,
operation='list',
wsType='',
uri='/',
name='',
arguments=None,
params=None):
if arguments is None:
arguments = {}
if params is None:
params = {}
rq = Request(operationName=operation, locale='fr_FR')
for k in arguments:
rq.append(RequestArgument(name=k, value=arguments[k]))
# Add resource descriptor
rd = RequestRD(operation, name, uri)
label = etree.SubElement(rd, 'label')
label.text = 'null'
# Add query parameters
for k, v in params.items():
p = etree.SubElement(rd, 'parameter', name=k)
if isinstance(v, basestring):
p.text = v
else:
p.text = str(v)
rq.append(rd)
return etree.tostring(rq, pretty_print=True)
def run_report(self, uri='/', output='PDF', params=None):
"""
Launch a runReport in Jasper
"""
if output not in KNOWN_FORMAT:
raise UnknownFormat(output)
args = {
'RUN_OUTPUT_FORMAT': output,
'PAGE': '0',
}
return self.create_request(operation='runReport',
wsType='reportUnit',
uri=uri,
arguments=args,
params=params)
@staticmethod
def parseMultipart(res):
srch = re.search(r'----=[^\r\n]*', res)
if srch is None:
raise NotMultipartContent()
boundary = srch.group()
res = " \n" + res
res = "Content-Type: multipart/alternative; boundary=%s\n%s" % \
(boundary, res)
message = email.message_from_string(res)
attachment = message.get_payload()[1]
return {
'content-type': attachment.get_content_type(),
'data': attachment.get_payload()
}
def log_last_request(self, ):
"""
Return the last SOAP query as text
"""
return self.body
def main():
app = make_app()
net_valid = False
conn_attempt = 0
if not args.no_prov:
if not args.no_check_ip:
net_valid = False
conn_attempt = 0
# fuzzy-check if machine is connected to ITEAD AP
while True:
conn_attempt += 1
for iface in netifaces.interfaces():
# if not
# netifaces.ifaddresses(iface).has_key(netifaces.AF_INET):
# python2
if netifaces.AF_INET not in netifaces.ifaddresses(
iface): # python3
continue
for afinet in netifaces.ifaddresses(iface)[
netifaces.AF_INET]:
# print(afinet['addr'])
if afinet['addr'].startswith("10.10.7."):
net_valid = True
break
if not net_valid:
try:
raise Exception(
"IPADDR_ITEAD_NOT_ASSIGNED",
"You do not appear to be connected to the ITEAD device"
)
except Exception as e:
if (e.args[0] != "IPADDR_ITEAD_NOT_ASSIGNED"):
raise
if conn_attempt == 1:
print(
"** No ip address of the ITEAD DHCP range (10.10.7.0/24)",
"is assigned to any of your interfaces,",
"which means you don't appear to be connected to the IEAD WiFi network."
)
print(
"** Please change into the ITEAD WiFi network (ITEAD-100001XXXX)"
)
print("** This application can be kept running.")
sleep(2)
print(".", end="", flush=True)
continue
else:
break
http = Http(timeout=2)
print("~~ Connection attempt")
conn_attempt = 0
while True:
conn_attempt += 1
print(">> HTTP GET /10.10.7.1/device")
try:
resp, cont = http.request("http://10.10.7.1/device", "GET")
break
except socket_error as e:
print(e)
continue
dct = json.loads(cont.decode('utf-8'))
print("<< %s" % json.dumps(dct, indent=4))
data = {
"version": 4,
"ssid": args.wifi_ssid,
"password": args.wifi_password,
"serverName": args.serving_host,
"port": default_port
}
print(">> HTTP POST /10.10.7.1/ap")
print(">> %s", json.dumps(data, indent=4))
resp, cont = http.request("http://10.10.7.1/ap", "POST",
json.dumps(data))
dct = json.loads(cont.decode('utf-8'))
print("<< %s" % json.dumps(dct, indent=4))
print("~~ Provisioning completed")
if not args.no_check_ip:
net_valid = False
conn_attempt = 0
# check if machine has <args.serving_host> being assigned on any iface
while True:
conn_attempt += 1
for iface in netifaces.interfaces():
# if not
# netifaces.ifaddresses(iface).has_key(netifaces.AF_INET): #
# python2
if netifaces.AF_INET not in netifaces.ifaddresses(
iface): # python3
continue
for afinet in netifaces.ifaddresses(iface)[netifaces.AF_INET]:
# print(afinet['addr'])
if afinet['addr'] == args.serving_host:
net_valid = True
break
if not net_valid:
try:
raise Exception(
"IPADDR_SRVHOST_NOT_ASSIGNED",
"The IP address of <serving_host> (%s) is not assigned to any interface"
% args.serving_host)
except Exception as e:
if (e.args[0] != "IPADDR_SRVHOST_NOT_ASSIGNED"):
raise
if conn_attempt == 1:
print(
"** The IP address of <serve_host> (%s) is not " %
args.serving_host,
"assigned to any interface on this machine.")
print(
"** Please change WiFi network to $ESSID and make ",
"sure %s is being assigned to your WiFi interface."
)
print("** This application can be kept running.")
sleep(2)
print(".", end="", flush=True)
continue
else:
break
print("~~ Starting web server")
if args.legacy:
old = make_app()
# listening on port 8081 for serving upgrade files for older devices
old.listen(8081)
# listening on port 8080 for serving upgrade files
app.listen(8080)
app_ssl = tornado.httpserver.HTTPServer(app,
ssl_options={
"certfile": "ssl/server.crt",
"keyfile": "ssl/server.key",
})
# listening on port 443 to catch initial POST request to eu-disp.coolkit.cc
app_ssl.listen(default_port)
print("~~ Waiting for device to connect")
tornado.ioloop.IOLoop.instance().start()
def _sent_form(self):
http = Http()
data = urlencode(self._form_values)
return http.request(self._url, "POST", data)
class ClusterWithGlusterfs:
SUCEESS = 0
ERROR_OTHER = 1
ERROR_NO_SUCH_ACTION_SUPPORT = 2
ERROR_NO_ACTION_PARAMETER = 3
ERROR_PARAMETER_MISSING = 4
ERROR_SIZE_LESS_THAN_ONE = 5
ERROR_SIZE_DIFFERENT_FROM_GLUSTERFS_PATH_AMOUNT = 6
ERROR_FAIL_TO_CREATE_REPLICATION_CONTROLLER = 7
ERROR_FAIL_TO_CREATE_SEED_INSTANCE = 8
ERROR_FAIL_TO_CREATE_SERVICE = 9
ERROR_FAIL_TO_CREATE_JOINING_INSTANCE = 10
ERROR_FAIL_TO_GET_OWNING_REPLICATION_CONTROLLER_LIST = 11
ERROR_FAIL_TO_DELETE_SERVICE = 12
ERROR_FAIL_TO_DELETE_REPLICATION_CONTROLLER = 13
ERROR_FAIL_TO_DELETE_POD = 14
def __init__(self):
self.parameter_dictionary = self.__get_input()
self.action = self.parameter_dictionary.get("action")
if self.action == None:
sys.exit(ClusterWithGlusterfs.ERROR_NO_ACTION_PARAMETER)
if self.action == "create":
self.parameter_list = [
"application_name",
"kube_apiserver_endpoint",
"kube_apiserver_token",
"namespace",
"size",
"service_file_name",
"replication_controller_file_name",
"environment_file_name",
"timeout_in_second",
"action",
]
if self.__check_input() is False:
sys.exit(ClusterWithGlusterfs.ERROR_PARAMETER_MISSING)
else:
print "Parameters are: " + str(self.parameter_dictionary)
self.__initialize_create()
elif self.action == "resize":
self.parameter_list = [
"application_name",
"kube_apiserver_endpoint",
"kube_apiserver_token",
"namespace",
"size",
"replication_controller_file_name",
"environment_file_name",
"timeout_in_second",
"action",
]
if self.__check_input() is False:
sys.exit(ClusterWithGlusterfs.ERROR_PARAMETER_MISSING)
else:
print "Parameters are: " + str(self.parameter_dictionary)
self.__initialize_resize()
elif self.action == "delete":
self.parameter_list = [
"application_name",
"kube_apiserver_endpoint",
"kube_apiserver_token",
"namespace",
"timeout_in_second",
"action",
]
if self.__check_input() is False:
sys.exit(ClusterWithGlusterfs.ERROR_PARAMETER_MISSING)
else:
print "Parameters are: " + str(self.parameter_dictionary)
self.__initialize_delete()
def __initialize_create(self):
self.http = Http(disable_ssl_certificate_validation=True)
# Parameter
self.application_name = self.parameter_dictionary.get(
"application_name") #name
self.kube_apiserver_endpoint = self.parameter_dictionary.get(
"kube_apiserver_endpoint") #"http://127.0.0.1:8080
self.kube_apiserver_token = self.parameter_dictionary.get(
"kube_apiserver_token")
self.namespace = self.parameter_dictionary.get("namespace") #default
self.size = int(self.parameter_dictionary.get("size")) #3
self.service_file_name = self.parameter_dictionary.get(
"service_file_name") #"service.json"
self.replication_controller_file_name = self.parameter_dictionary.get(
"replication_controller_file_name") #"replication-controller.json"
self.environment_file_name = self.parameter_dictionary.get(
"environment_file_name") #"environment.json"
self.time_to_wait = int(
self.parameter_dictionary.get("timeout_in_second")) #60 * 3
self.action = self.parameter_dictionary.get("action") #create
# File data
self.replication_controller_dictionary = self.__load_replication_controller_file(
)
self.service_dictionary = self.__load_service_file()
self.environment_pair_list = self.__load_environment_file()
# Indirect Data
self.service_name = self.application_name #self.service_dictionary.get("metadata").get("name")
self.replication_controller_name = self.application_name + "-instance" #self.replication_controller_dictionary.get("metadata").get("name")
self.glusterfs_path_list = self.__get_glusterfs_path_list()
self.glusterfs_endpoints = self.__get_glusterfs_endpoints()
def __initialize_resize(self):
self.http = Http(disable_ssl_certificate_validation=True)
# Parameter
self.application_name = self.parameter_dictionary.get(
"application_name") #name
self.kube_apiserver_endpoint = self.parameter_dictionary.get(
"kube_apiserver_endpoint") #"http://127.0.0.1:8080
self.kube_apiserver_token = self.parameter_dictionary.get(
"kube_apiserver_token")
self.namespace = self.parameter_dictionary.get("namespace") #default
self.size = int(self.parameter_dictionary.get("size")) #3
self.replication_controller_file_name = self.parameter_dictionary.get(
"replication_controller_file_name") #"replication-controller.json"
self.environment_file_name = self.parameter_dictionary.get(
"environment_file_name") #"environment.json"
self.time_to_wait = int(
self.parameter_dictionary.get("timeout_in_second")) #60 * 3
self.action = self.parameter_dictionary.get("action") #create
# File data
self.replication_controller_dictionary = self.__load_replication_controller_file(
)
self.environment_pair_list = self.__load_environment_file()
# Indirect Data
self.service_name = self.application_name #self.service_dictionary.get("metadata").get("name")
self.replication_controller_name = self.application_name + "-instance" #self.replication_controller_dictionary.get("metadata").get("name")
self.glusterfs_path_list = self.__get_glusterfs_path_list()
self.glusterfs_endpoints = self.__get_glusterfs_endpoints()
def __initialize_delete(self):
self.http = Http(disable_ssl_certificate_validation=True)
# Parameter
self.application_name = self.parameter_dictionary.get(
"application_name") #name
self.kube_apiserver_endpoint = self.parameter_dictionary.get(
"kube_apiserver_endpoint") #"http://127.0.0.1:8080
self.kube_apiserver_token = self.parameter_dictionary.get(
"kube_apiserver_token")
self.namespace = self.parameter_dictionary.get("namespace") #default
self.time_to_wait = int(
self.parameter_dictionary.get("timeout_in_second")) #60 * 3
self.action = self.parameter_dictionary.get("action") #create
# Indirect Data
self.service_name = self.application_name #self.service_dictionary.get("metadata").get("name")
self.replication_controller_name = self.application_name + "-instance" #self.replication_controller_dictionary.get("metadata").get("name")
def __get_glusterfs_path_list(self):
for environment_pair in self.environment_pair_list:
if environment_pair.get("name") == "GLUSTERFS_PATH_LIST":
glusterfs_path_list = environment_pair.get("value").split(",")
for i in xrange(0, len(glusterfs_path_list)):
glusterfs_path_list[i] = glusterfs_path_list[i].strip()
return glusterfs_path_list
return []
def __get_glusterfs_endpoints(self):
for environment_pair in self.environment_pair_list:
if environment_pair.get("name") == "GLUSTERFS_ENDPOINTS":
return environment_pair.get("value")
return None
def __get_input(self):
parameter_dictionary = dict()
parameter_list = sys.argv[1:]
for parameter in parameter_list:
key_value = parameter[2:]
key_value_list = key_value.split("=")
parameter_dictionary[key_value_list[0]] = key_value_list[1]
return parameter_dictionary
def __check_input(self):
result = True
for parameter in self.parameter_list:
if self.parameter_dictionary.get(parameter) == None:
print "Parameter " + parameter + " is missing"
result = False
return result
def __load_replication_controller_file(self):
with open(self.replication_controller_file_name, "r") as file_read:
text = file_read.read()
return json.loads(text)
def __load_service_file(self):
with open(self.service_file_name, "r") as file_read:
text = file_read.read()
return json.loads(text)
def __load_environment_file(self):
with open(self.environment_file_name, "r") as file_read:
text = file_read.read()
return json.loads(text)
# Check seed instance
def __check_seed_instance_up(self, pod_list):
for pod in pod_list:
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/pods/" + pod + "/log",
"GET",
headers={"Authorization": self.kube_apiserver_token})
for seed_instance_up_keyword in self.seed_instance_up_keyword_list:
if seed_instance_up_keyword in body:
pass
else:
return None
return True
# Check joining nodes are up in the cluster
def __check_joining_instance_up(self, pod_list):
for pod in pod_list:
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/pods/" + pod + "/log",
"GET",
headers={"Authorization": self.kube_apiserver_token})
for joining_instance_up_keyword in self.joining_instance_up_keyword_list:
if joining_instance_up_keyword in body:
pass
else:
return None
for joining_node_failure_keyword in self.joining_node_failure_keyword_list:
if joining_node_failure_keyword in body:
print body
return False
return True
# Check service is up
def __check_service_up(self):
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/services/" + self.service_name,
"GET",
headers={"Authorization": self.kube_apiserver_token})
if head.status == 200:
return True
else:
return None
# Get all pod name
def __get_all_pod_name_in_replication_controller(
self, expected_size, replication_controller_number):
pod_list = []
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/pods/",
"GET",
headers={"Authorization": self.kube_apiserver_token})
dictionary = json.loads(body)
for item in dictionary.get("items"):
# Remove the pod name part -xxxxx
name = item.get("metadata").get("name")[:-6]
if name == self.__get_replication_controller_instance_name(
replication_controller_number):
pod_list.append(item.get("metadata").get("name"))
if expected_size == -1 or expected_size == len(pod_list):
return pod_list
else:
return None
def __get_replication_controller_instance_name(
self, replication_controller_number):
return self.replication_controller_name + "-" + str(
replication_controller_number)
def __create_replication_controller(self, replication_controller_number):
replication_controller_dictionary = copy.deepcopy(
self.replication_controller_dictionary)
replication_controller_dictionary["metadata"][
"name"] = self.__get_replication_controller_instance_name(
replication_controller_number)
replication_controller_dictionary["metadata"]["labels"][
"name"] = self.__get_replication_controller_instance_name(
replication_controller_number)
replication_controller_dictionary["spec"]["selector"][
"name"] = self.__get_replication_controller_instance_name(
replication_controller_number)
replication_controller_dictionary["spec"]["template"]["metadata"][
"labels"][
"name"] = self.__get_replication_controller_instance_name(
replication_controller_number)
replication_controller_dictionary["spec"]["selector"][
"group"] = self.service_name
replication_controller_dictionary["spec"]["template"]["metadata"][
"labels"]["group"] = self.service_name
volume_list = replication_controller_dictionary["spec"]["template"][
"spec"]["volumes"]
for volume in volume_list:
if volume.get("name") == self.volume_to_mount:
volume["glusterfs"]["endpoints"] = self.glusterfs_endpoints
volume["glusterfs"]["path"] = self.glusterfs_path_list[
replication_controller_number]
return replication_controller_dictionary
def __create_service(self):
service_dictionary = copy.deepcopy(self.service_dictionary)
service_dictionary["metadata"]["name"] = self.service_name
service_dictionary["metadata"]["labels"]["name"] = self.service_name
service_dictionary["spec"]["selector"]["group"] = self.service_name
return service_dictionary
def __create_replication_controller_and_check(
self, replication_controller_number, check_function):
# Create a replication controller
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/replicationcontrollers",
"POST",
json.dumps(
self.__create_replication_controller(
replication_controller_number)),
headers={"Authorization": self.kube_apiserver_token})
if head.status != 201:
print "Fail to create replication controller " + self.__get_replication_controller_instance_name(
replication_controller_number)
print head
print body
return ClusterWithGlusterfs.ERROR_FAIL_TO_CREATE_REPLICATION_CONTROLLER
# Get pod name
pod_list = Utility.execute_until_timeout(
self.__get_all_pod_name_in_replication_controller,
self.time_to_wait, 1, replication_controller_number)
if pod_list is False:
print("Fail to get pod name for replication controller number " +
str(replication_controller_number))
return False
else:
print("The created pod for replication controller number " +
str(replication_controller_number) + " is: " +
str(pod_list))
# Check pod
if Utility.execute_until_timeout(check_function, self.time_to_wait,
pod_list):
print("Successfully create the instance")
else:
print("Instance fail to come up")
return False
def __get_owning_replication_controller_list(self):
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/replicationcontrollers",
"GET",
headers={"Authorization": self.kube_apiserver_token})
if head.status == 200:
owning_replication_controller_list = []
dictionary = json.loads(body)
item_list = dictionary.get("items")
for item in item_list:
selector_group = item.get("spec").get("selector").get("group")
selector_name = item.get("spec").get("selector").get("name")
if selector_group == self.service_name and selector_name.startswith(
self.replication_controller_name):
owning_replication_controller_list.append(
item.get("metadata").get("name"))
return owning_replication_controller_list, ClusterWithGlusterfs.SUCEESS
else:
print "Fail to get owning replication controller list"
print head
print body
return None, ClusterWithGlusterfs.ERROR_FAIL_TO_GET_OWNING_REPLICATION_CONTROLLER_LIST
def create_cluster(self):
# Check
if self.size < 1:
print("Size " + str(self.size) + " can't be less than 1")
return ClusterWithGlusterfs.ERROR_SIZE_LESS_THAN_ONE
if len(self.glusterfs_path_list) != self.size:
print("Size " + str(self.size) +
" is not the same as the path list " +
str(self.glusterfs_path_list) + " amount " +
str(len(self.glusterfs_path_list)))
return ClusterWithGlusterfs.ERROR_SIZE_DIFFERENT_FROM_GLUSTERFS_PATH_AMOUNT
# Create and check seed instance
if self.__create_replication_controller_and_check(
0, self.__check_seed_instance_up) is False:
print("Fail to create seed instance")
return ClusterWithGlusterfs.ERROR_FAIL_TO_CREATE_SEED_INSTANCE
# Create a service to track joining instances
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/services",
"POST",
json.dumps(self.__create_service()),
headers={"Authorization": self.kube_apiserver_token})
if head.status != 201:
print "Fail to create service " + self.service_name
print head
print body
return ClusterWithGlusterfs.ERROR_FAIL_TO_CREATE_SERVICE
# Check service
if Utility.execute_until_timeout(self.__check_service_up,
self.time_to_wait):
print("Successfully create the service")
else:
print("Fail to create service")
return ClusterWithGlusterfs.ERROR_FAIL_TO_CREATE_SERVICE
# Add other instances
if self.size > 1:
for i in xrange(1, self.size):
if self.__create_replication_controller_and_check(
i, self.__check_joining_instance_up) is False:
print("Fail to create joining instance " + str(i))
return ClusterWithGlusterfs.ERROR_FAIL_TO_CREATE_JOINING_INSTANCE
return ClusterWithGlusterfs.SUCEESS
else:
return ClusterWithGlusterfs.SUCEESS
def resize_cluster(self):
owning_replication_controller_list, status_code = self.__get_owning_replication_controller_list(
)
if status_code != ClusterWithGlusterfs.SUCEESS:
return status_code
current_size = len(owning_replication_controller_list)
if self.size == current_size:
print "Size is not changed"
return ClusterWithGlusterfs.SUCEESS
elif self.size < current_size:
# Delete replication controllers and related pods
for i in xrange(self.size, current_size):
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/replicationcontrollers/" +
self.__get_replication_controller_instance_name(i),
"DELETE",
headers={"Authorization": self.kube_apiserver_token})
if head.status != 200:
print "Fail to delete replication controller " + self.__get_replication_controller_instance_name(
i)
print head
print body
return ClusterWithGlusterfs.ERROR_FAIL_TO_DELETE_REPLICATION_CONTROLLER
pod_list = Utility.execute_until_timeout(
self.__get_all_pod_name_in_replication_controller,
self.time_to_wait, -1, i)
for pod in pod_list:
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/pods/" + pod,
"DELETE",
headers={"Authorization": self.kube_apiserver_token})
if head.status != 200:
print "Fail to delete pod " + pod
print head
print body
return ClusterWithGlusterfs.ERROR_FAIL_TO_DELETE_POD
return ClusterWithGlusterfs.SUCEESS
elif self.size > current_size:
# Create replication controllers
for i in xrange(current_size, self.size):
if self.__create_replication_controller_and_check(
i, self.__check_joining_instance_up) is False:
print("Fail to create joining instance " + str(i))
return ClusterWithGlusterfs.ERROR_FAIL_TO_CREATE_JOINING_INSTANCE
return ClusterWithGlusterfs.SUCEESS
def clean_cluster(self):
owning_replication_controller_list, status_code = self.__get_owning_replication_controller_list(
)
if status_code != ClusterWithGlusterfs.SUCEESS:
return status_code
size = len(owning_replication_controller_list)
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/services/" + self.service_name,
"DELETE",
headers={"Authorization": self.kube_apiserver_token})
if head.status != 200:
print "Fail to delete service " + self.service_name
print head
print body
return ClusterWithGlusterfs.ERROR_FAIL_TO_DELETE_SERVICE
for i in xrange(0, size):
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/replicationcontrollers/" +
self.__get_replication_controller_instance_name(i),
"DELETE",
headers={"Authorization": self.kube_apiserver_token})
if head.status != 200:
print "Fail to delete replication controller " + self.__get_replication_controller_instance_name(
i)
print head
print body
return ClusterWithGlusterfs.ERROR_FAIL_TO_DELETE_REPLICATION_CONTROLLER
pod_list = Utility.execute_until_timeout(
self.__get_all_pod_name_in_replication_controller,
self.time_to_wait, -1, i)
for pod in pod_list:
head, body = self.http.request(
self.kube_apiserver_endpoint + "/api/v1/namespaces/" +
self.namespace + "/pods/" + pod,
"DELETE",
headers={"Authorization": self.kube_apiserver_token})
if head.status != 200:
print "Fail to delete pod " + pod
print head
print body
return ClusterWithGlusterfs.ERROR_FAIL_TO_DELETE_POD
return ClusterWithGlusterfs.SUCEESS
{
'type': 'image',
'duplicates': duplicated_image
},
{
'type': 'address',
'duplicates': duplicated_address
}
],
'positive_words': positive_words,
'negative_words': negative_words
})
http_obj = Http()
resp, content = http_obj.request(
uri=uiListingUrl,
method='POST',
headers={'Content-Type': 'application/json; charset=UTF-8'},
body=json_resp
)
return resp
except Exception, e:
return Response(json.dumps({'error': str(e)}), status=500, mimetype='application/json')
@app.route('/api/fraud/search', methods=['GET'])
def calculate_scraped_listing_score():
crawler_listing_id = request.args.get('crawler_listing_id')
print 'listing id is: %s' % crawler_listing_id
email_address = request.args.get('email_address')
calculateScore.apply_async(args=[crawler_listing_id, email_address])
return json.dumps({"status": "updating"})
def run(self):
while True:
proxy_num = select_proxy()
proxy = proxy_list[proxy_num]
try:
temp = 0
global start_count
if start_count >= len(combos):
break
with lock:
k = start_count
start_count += 1
print(start_count)
combo = combos[k]
http = Http(
proxy_info=ProxyInfo(proxy_host=str(proxy["proxy"]), proxy_port=int(proxy["port"]),
proxy_type=proxy["type"], proxy_user=proxy["user"], proxy_pass=proxy["pass"]))
headers = {
"User-Agent": "Dalvik/2.1.0 (Linux; U; Android 5.1.1; SM-G955N Build/NRD90M)",
"Content-Type": "application/json",
"Origin": "https://www.hotstar.com",
"Referer": "https://www.hotstar.com/in",
"hotstarauth": "st=1556707390~exp=1556707990~acl=/*~hmac=ad817d9e5b9c219b7cdb086e891e8633187c3723f6aa1212fda9b4140e60036f"
}
body = json.dumps({
"isProfileRequired": True,
"userData": {
"deviceId": "c3f61ee7-16e7-415a-aeae-43dfea1eda87",
"pId": "a25865a67413447098a8d1ee7388982e",
"password": combo["password"],
"username": combo["email"],
"usertype": "email"
},
"verification": {}
})
res, cont = http.request("https://api.hotstar.com/in/aadhar/v2/android/in/users/login", method="POST",
body=body, headers=headers)
if cont:
cont = json.loads(cont.decode(encoding="UTF-8"))
headers["userId"] = cont["description"]["userIdentity"]
headers["hotstarauth"] = "st=1556707980~exp=1556708580~acl=/in/gringotts*~hmac=d7abcbad27cdf9427c10cdc51da81551dbd949a82898539c97a894c933b30b24"
res, cont = http.request("https://api.hotstar.com/in/gringotts/v2/android/in/subscription?verbose=1",
method="GET", headers=headers)
cont = json.loads(cont.decode(encoding="UTF-8"))
print(cont["active_subs"][0]["commercial_pack"])
if "active_subs" in cont.keys():
print(combo, "Expires On: ", cont["active_subs"][0]["commercial_pack"])
combo["expireOn"] = cont["active_subs"][0]["commercial_pack"]
verified.append(combo)
# with lock:
with open("hotstar"+str(k) + ".json", "w") as a:
json.dump(combo, a)
else:
print(combo, " Free")
combo["expireOn"] = "Free"
verified.append(combo)
with open("Free-" + str(k) + ".json", "w") as a:
json.dump(combo, a)
del http
except Exception as a:
# print(a)
inc_count(proxy_num)
continue
if temp == 1:
break
from httplib2 import Http
from urllib import urlencode
import re
print "\t################################################################"
print "\t# Kapi #"
print "\t# Scan Links On a Web Page #"
print "\t################################################################"
site = raw_input("Enter Site name including http, https: ")
h = Http()
resp, content = h.request(
site,
"GET",
headers={
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0'
})
#print content
links = re.findall(r"<a.*?\s*href=\"(.*?)\".*?>(.*?)</a>", content)
f = open('result.txt', 'w')
for link in links:
print(link[0])
f.write(link[0] + "\n") # python will convert \n to os.linesep
f.close() # you can omit in most cases as the destructor will call if
print "*** Result on result.txt"
def sendMessage(self, event):
url = self.webhook
status = event[0]
# Set card title and image
stat = None
if status == "0":
stat = "Problem"
image_url = self.PROBLEM_IMG
elif status == "1":
stat = "Resolved"
image_url = self.RESOLVED_IMG
elif status == "2":
stat = "Acknowledged"
image_url = self.ACK_IMG
# If it is a problem or resolution message
if status == "0" or status == "1":
time = event[1]
date = event[2]
trigger_name = event[3]
host_name = event[4]
severity = event[5]
self.event_id = event[6]
trigger_url = event[7]
self.trigger_id = event[8]
host_description = event[9]
bot_message = {
"cards": [{
"header": {
"title": "Severity: " + severity,
"subtitle": stat,
"imageUrl": image_url,
"imageStyle": "IMAGE"
},
"sections": [{
"widgets": [{
"keyValue": {
"topLabel": "Alarm",
"content": trigger_name,
"contentMultiline": "true"
}
}, {
"keyValue": {
"topLabel": "Host",
"content": host_name + " " + host_description,
"contentMultiline": "true"
}
}, {
"keyValue": {
"topLabel": "Date/Time",
"content": date + " - " + time
}
}, {
"keyValue": {
"topLabel": "Event ID",
"content": self.event_id
}
}]
}, {
"widgets": [{
"buttons": [{
"textButton": {
"text": "View event on ZABBIX",
"onClick": {
"openLink": {
"url":
self.zabbix_url +
"/tr_events.php?triggerid=" +
self.trigger_id + "&eventid=" +
self.event_id
}
}
}
}]
}]
}]
}]
}
# If it is an acknowledgment message
elif status == "2":
time = event[1]
date = event[2]
ack_user = event[3]
ack_message = event[4]
event_status = event[5]
self.event_id = event[6]
self.trigger_id = event[7]
# if event_status == "PROBLEM":
# event_status = "Ativo"
# elif event_status == "RESOLVED":
# event_status = "Resolvido"
bot_message = {
"cards": [{
"header": {
"title": stat,
"subtitle": ack_user,
"imageUrl": image_url,
"imageStyle": "IMAGE"
},
"sections": [{
"widgets": [{
"keyValue": {
"topLabel": "Message",
"content": ack_message,
"contentMultiline": "true"
}
}, {
"keyValue": {
"topLabel": "Current Alarm Status",
"content": event_status
}
}, {
"keyValue": {
"topLabel": "Date/Time",
"content": date + " - " + time
}
}, {
"keyValue": {
"topLabel": "Event ID",
"content": self.event_id
}
}]
}, {
"widgets": [{
"buttons": [{
"textButton": {
"text": "View event on ZABBIX",
"onClick": {
"openLink": {
"url":
self.zabbix_url +
"/tr_events.php?triggerid=" +
self.trigger_id + "&eventid=" +
self.event_id
}
}
}
}]
}]
}]
}]
}
# Check if Event already has a thread
if self.trigger_id in self.evt_thread:
self.thread = self.evt_thread[self.trigger_id]
bot_message['thread'] = {"name": self.thread}
message_headers = {'Content-Type': 'application/json; charset=UTF-8'}
# HTTP API Request
http_obj = Http()
response = http_obj.request(
uri=url,
method='POST',
headers=message_headers,
body=dumps(bot_message),
)
# Take the request response thread and store
self.thread = json.loads(response[1])['thread']['name']
event_thread = {self.trigger_id: self.thread}
self.writeEventThread(event_thread)
from geventhttpclient import httplib
httplib.patch()
from httplib2 import Http
http = Http()
response, content = http.request('http://google.fr/')
assert response.status == 200
assert content
print response
print content
response, content = http.request('http://google.fr/', method='HEAD')
assert response.status == 200
assert content == ''
print response
response, content = http.request('https://www.google.com/', method='HEAD')
assert response.status == 200
assert content == ''
print response
class HttpApiClient(object):
"""
Base implementation for an HTTP
API Client. Used by the different
API implementation objects to manage
Http connection.
"""
def __init__(self, api_key, base_url):
"""Initialize base http client."""
self.conn = Http()
# MenuPlatform API key
self.api_key = api_key
#base url
self.base_url = base_url
def _http_request(self, service_type, **kwargs):
"""
Perform an HTTP Request using base_url and parameters
given by kwargs.
Results are expected to be given in JSON format
and are parsed to python data structures.
"""
request_params = urlencode(kwargs)
request_params = request_params.replace('%28', '').replace('%29', '')
uri = '%s%s?api_key=%s&%s' % \
(self.base_url, service_type, self.api_key, request_params)
header, response = self.conn.request(uri, method='GET')
return header, response
def _http_uri_request(self, uri):
header, response = self.conn.request(uri, method='GET')
return header, response
def _is_http_response_ok(self, response):
return response['status'] == '200' or response['status'] == 200
def _get_params(self, name = None, category = None, cuisine = None, description = None, price = None, \
price__gt = None, price__gte = None, price__lt = None, price__lte = None, \
location = (None, None), radius = None, tl_coord = (None, None), \
br_coord = (None, None), country = None, locality = None, \
region = None, postal_code = None, street_address = None, website_url = None, dimension = None, has_menu = None, open_at = None):
lat, long = location
tl_lat, tl_long = tl_coord
br_lat, br_long = br_coord
params = {}
if name:
params['name'] = name
if category:
if not isinstance(category, list):
raise TypeError(
'Please provide list of categories as a category parameter'
)
params['category'] = ','.join(category)
if cuisine:
if not isinstance(cuisine, list):
raise TypeError(
'Please provide list of cuisines as a cuisines parameter')
params['cuisine'] = ','.join(cuisine)
if description:
params['description'] = description
if price:
params['price'] = price
if price__gt:
params['price__gt'] = price__gt
if price__gte:
params['price__gte'] = price__gte
if price__lt:
params['price__lt'] = price__lt
if price__lte:
params['price__lte'] = price__lte
if lat and long:
params['location'] = '%s,%s' % (lat, long)
if radius:
params['radius'] = radius
if tl_lat and tl_long and br_lat and br_long:
params['bounds'] = '%s,%s|%s,%s' % (tl_lat, tl_long, br_lat,
br_long)
if country:
params['country'] = country
if locality:
params['locality'] = locality
if region:
params['region'] = region
if postal_code:
params['postal_code'] = postal_code
if street_address:
params['street_address'] = street_address
if website_url:
params['website_url'] = website_url
if dimension:
params['dimension'] = dimension
if has_menu != None:
params['has_menu'] = has_menu
if open_at:
params['open_at'] = open_at
return params
def _create_query(self, category_type, params):
header, content = self._http_request(category_type + '/', **params)
resp = json.loads(content)
if not self._is_http_response_ok(header):
error = resp.get('error_message', 'Unknown Error')
raise HttpException(header.status, header.reason, error)
return resp
class TrelloClient(object):
""" Base class for Trello API access """
def __init__(self,
api_key,
api_secret=None,
token=None,
token_secret=None):
"""
Constructor
:api_key: API key generated at https://trello.com/1/appKey/generate
:api_secret: the secret component of api_key
:token_key: OAuth token generated by the user in
trello.util.create_oauth_token
:token_secret: the OAuth client secret for the given OAuth token
"""
if api_key and api_secret and token and token_secret:
# oauth
self.oauth_consumer = oauth.Consumer(key=api_key,
secret=api_secret)
self.oauth_token = oauth.Token(key=token, secret=token_secret)
self.client = oauth.Client(self.oauth_consumer, self.oauth_token)
elif api_key:
self.client = Http()
if token is None:
self.public_only = True
else:
self.public_only = False
self.api_key = api_key
self.auth_token = token
def info_for_all_boards(self, actions):
"""
Use this if you want to retrieve info for all your boards in one swoop
"""
if self.public_only:
return None
else:
json_obj = self.fetch_json('/members/me/boards/all',
query_params={'actions': actions})
self.all_info = json_obj
def logout(self):
"""Log out of Trello."""
#TODO: This function.
raise NotImplementedError()
def build_url(self, path, query={}):
"""
Builds a Trello URL.
:path: URL path
:params: dict of key-value pairs for the query string
"""
url = 'https://api.trello.com/1'
if path[0:1] != '/':
url += '/'
url += path
if hasattr(self, 'oauth_token'):
url += '?'
url += "key=" + self.oauth_consumer.key
url += "&token=" + self.oauth_token.key
else:
url += '?'
url += "key=" + self.api_key
if self.public_only is False:
url += "&token=" + self.auth_token
if len(query) > 0:
url += '&' + urlencode(query)
return url
def list_boards(self):
"""
Returns all boards for your Trello user
:return: a list of Python objects representing the Trello boards.
Each board has the following noteworthy attributes:
- id: the board's identifier
- name: Name of the board
- desc: Description of the board (optional - may be missing from the
returned JSON)
- closed: Boolean representing whether this board is closed or not
- url: URL to the board
"""
json_obj = self.fetch_json('/members/me/boards')
boards = list()
for obj in json_obj:
boards.append(self._board_from_json(obj))
return boards
def get_board(self, board_id):
obj = self.fetch_json('/boards/' + board_id)
return self._board_from_json(obj)
def add_board(self, board_name):
obj = self.fetch_json('/boards',
http_method='POST',
post_args={'name': board_name})
board = Board(self, obj['id'], name=obj['name'].encode('utf-8'))
board.closed = obj['closed']
return board
def get_list(self, list_id):
obj = self.fetch_json('/lists/' + list_id)
list = List(self.get_board(obj['idBoard']),
obj['id'],
name=obj['name'].encode('utf-8'))
list.closed = obj['closed']
return list
def get_member(self, member_id):
return Member(self, member_id).fetch()
def fetch_json(self,
uri_path,
http_method='GET',
headers={},
query_params={},
post_args={}):
""" Fetch some JSON from Trello """
if http_method in ("POST", "PUT", "DELETE"):
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json'
url = self.build_url(uri_path, query_params)
response, content = self.client.request(url,
http_method,
headers=headers,
body=json.dumps(post_args))
# error checking
if response.status == 401:
raise Unauthorized(url, response)
if response.status != 200:
raise ResourceUnavailable(url, response)
return json.loads(content)
def _board_from_json(self, json):
board = Board(self, json['id'], name=json['name'].encode('utf-8'))
board.description = json.get('desc', '').encode('utf-8')
board.closed = json['closed']
board.url = json['url']
return board
def list_hooks(self, token=None):
"""
Returns a list of all hooks associated with a specific token. If you don't pass in a token,
it tries to use the token associated with the TrelloClient object (if it exists)
"""
if token is None and self.auth_token is None:
raise TokenError(
"You need to pass an auth token in to list hooks.")
else:
using_token = token if self.auth_token is None else self.auth_token
url = "/tokens/%s/webhooks" % using_token
return self._existing_hook_objs(self.fetch_json(url), using_token)
def _existing_hook_objs(self, hooks, token):
"""
Given a list of hook dicts passed from list_hooks, creates
the hook objects
"""
all_hooks = []
for hook in hooks:
new_hook = WebHook(self, token, hook['id'], hook['description'],
hook['idModel'], hook['callbackURL'],
hook['active'])
all_hooks.append(new_hook)
return all_hooks
def create_hook(self, callback_url, id_model, desc=None, token=None):
"""
Creates a new webhook. Returns the WebHook object created.
There seems to be some sort of bug that makes you unable to create a
hook using httplib2, so I'm using urllib2 for that instead.
"""
if token is None and self.auth_token is None:
raise TokenError(
"You need to pass an auth token in to create a hook.")
else:
using_token = token if self.auth_token is None else self.auth_token
url = "https://trello.com/1/tokens/%s/webhooks/?key=%s" % (
using_token, self.api_key)
data = urlencode({
'callbackURL': callback_url,
'idModel': id_model,
"description": desc
})
# TODO - error checking for invalid responses
# Before spending too much time doing that with urllib2, might be worth trying
# and getting it working with urllib2 for consistency
req = urllib2.Request(url, data)
response = urllib2.urlopen(req)
if response.code == 200:
hook_id = json.loads(response.read())['id']
return WebHook(self, using_token, hook_id, desc, id_model,
callback_url, True)
else:
return False
def make_request(self, url, method="GET", body=None, headers={}):
if not 'User-Agent' in headers:
headers.update(
{"User-Agent": "%s Python Client" % self.api.api_name})
http_obj = Http()
return http_obj.request(url, method, body=body, headers=headers)
address = raw_input(
"Please enter the address of the server you want to access, \n If left blank the connection will be set to 'http://localhost:5000': "
)
if address == '':
address = 'http://localhost:5000'
#TEST 1 TRY TO MAKE A NEW USER
try:
url = address + '/users'
h = Http()
#h.add_credentials('TinyTim', 'Udacity')
data = dict(username="******", password="******")
data = json.dumps(data)
resp, content = h.request(url,
'POST',
body=data,
headers={"Content-Type": "application/json"})
if resp['status'] != '201' and resp['status'] != '200':
raise Exception('Received an unsuccessful status code of %s' %
resp['status'])
except Exception as err:
print "Test 1 FAILED: Could not make a new user"
print err.args
sys.exit()
else:
print "Test 1 PASS: Succesfully made a new user"
#TEST 2 ADD NEW BAGELS TO THE DATABASE
try:
h = Http()
h.add_credentials('TinyTim', 'Udacity')
class Request(object):
def __init__(self, method, url, **opts):
self.method = method
self.url = url
self.headers = opts.pop('headers', {})
self.body = opts.pop('body', None)
self.disable_ssl = opts.pop('disable_ssl', True)
self.http = Http(disable_ssl_certificate_validation=self.disable_ssl)
self.content_type = 'application/x-www-form-urlencoded'
self.opts = opts
def __repr__(self):
return '<OAuth2 Request>'
def request(self):
parse = self.opts.pop('parse', 'json')
files = self.opts.pop('files', {})
params = urllib.urlencode(self.opts)
if self.method in ('POST', 'PUT'):
(body, content_type) = self.__encode_files(files, self.opts) if files else (params, self.content_type)
self.headers.update({'Content-Type': content_type})
self.body = body
elif self.opts:
self.url += '&%s'%params if '?' in self.url else '?%s'%params
response = self.send()
response = Response(response, parse=parse)
status = response.status
#TODO raise error
if status in (301, 302, 303, 307):
return response
elif 200 <= status < 400:
return response
elif 400 <= status < 500:
return response
elif 500 <= status < 600:
return response
return response
def send(self):
return self.http.request(self.url, self.method, body=self.body, headers=self.headers)
def __encode_files(self, files, params):
if not files or isinstance(files, str):
return None
fields = []
for k, v in tuples(files):
if isinstance(v, (tuple, list)):
fn, fp = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, str):
fp = StringIO(fp)
if isinstance(fp, bytes):
fp = BytesIO(fp)
fields.append((k, (fn, fp.read())))
for k, vs in tuples(params):
if isinstance(vs, list):
for v in vs:
fields.append((k, str(v)))
else:
fields.append((k, str(vs)))
body, content_type = build_multipart(fields)
return body, content_type
