def http_request(self, request):
data = request.get_data()
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if type(value) == file:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if request.has_header('Content-type'):
pass # print "Replacing %s with %s" % (request.get_header('content-type'), contenttype)
request.add_unredirected_header('Content-type', contenttype)
request.add_data(data)
return request
def http_request(self, request):
data = request.get_data()
if isinstance(data, dict):
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if isinstance(value, file) or hasattr(value, "file") or isinstance(value, io.StringIO):
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise SqlmapDataException("not a valid non-string sequence or mapping object").with_traceback(traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = "multipart/form-data; boundary=%s" % boundary
#if (request.has_header("Content-Type") and request.get_header("Content-Type").find("multipart/form-data") != 0):
# print "Replacing %s with %s" % (request.get_header("content-type"), "multipart/form-data")
request.add_unredirected_header("Content-Type", contenttype)
request.add_data(data)
return request
def http_request(self, request):
try:
data = request.get_data()
except AttributeError:
data = request.data
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for (key, value) in list(data.items()):
if hasattr(value, 'read'):
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
raise TypeError
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if (request.has_header('Content-Type') and request.get_header(
'Content-Type').find('multipart/form-data') != 0):
six.print_("Replacing %s with %s" %
(request.get_header('content-type'),
'multipart/form-data'))
request.add_unredirected_header('Content-Type', contenttype)
try:
request.add_data(data)
except AttributeError:
request.data = data
return request
def http_request(self, request):
data = request.get_data()
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if type(value) == io.BufferedReader:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
#raise TypeError("not a valid non-string sequence or mapping object %d" % traceback)
logger.debug("not a valid non-string sequence or mapping object : %s" % traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, self.doseq).encode()
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
request.add_unredirected_header('Content-Type', contenttype)
request.add_data(data)
return request
def http_request(self, request):
print(request.__dict__)
data = request.data
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for key, value in data.items():
if isinstance(value,IOBase):
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise TypeError("not a valid non-string sequence or mapping object")
if len(v_files) == 0:
data = urllib.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if(request.has_header('Content-Type')
and request.get_header('Content-Type').find('multipart/form-data') != 0):
print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data'))
request.add_unredirected_header('Content-Type', contenttype)
request.add_data(data)
return request
def test_post_local(self, filename, ccd_data, use_base64=False):
""" filename : The name of the file attachment
ccd_data : The xml to post (send)
use_base64 : If true, encode the encrypted XML as base64. Maybe necessary for 3DES
"""
attachments = ""
payload = """--%s
Content-Disposition: attachment; name="%s"; filename="%s.xml"
Content-Type: text/xml
%s"""
payload_uuid = str(uuid.uuid4())
#try:
for data in ccd_data:
if len(attachments) > 0:
attachments += "\r\n\r\n"
if use_base64:
attachments += payload % (payload_uuid, filename, filename, base64.b64encode(data))
else:
attachments += payload % (payload_uuid, filename, filename, data)
http = urllib.request.HTTPHandler(debuglevel=1)
opener = urllib.request.build_opener(http)
urllib.request.install_opener(opener)
request = urllib.request.Request(self._url)
request.add_header("Content-Type", "multipart/form-data; boundary=%s" % payload_uuid)
request.add_header("User-Agent", "synthesis")
request.add_data(attachments)
response = urllib.request.urlopen(request).read()
# check for some sign of success within the response
if response[0:4] == "202:":
return (True, response)
else:
return (False, response)
def glsrequest(uri, method, data=None):
'''
Returns xml node tree as Element instance.
'uri' may be absolute or relative to _BASEURI.
'method' in ('GET', 'POST', 'PUT')
'data' can be a string or Element instance
'''
if method not in {'GET', 'POST', 'PUT'}:
raise GlslibException(MSGUNSUPPORTEDMETHOD % method)
if not uri.startswith(_BASEURI):
uri = _BASEURI.rstrip('/') + '/' + uri.lstrip('/')
request = urllib.request.Request(uri)
request.add_header("Authorization", "Basic %s" % _AUTHSTR)
if etree.iselement(data):
# tostring generates bytestring (as required for data)
data = etree.tostring(data)
request.add_header('Content-Type', 'application/xml')
request.add_data(data)
request.get_method = lambda: method
msg = '%s %s\n%s\n%s' % (request.get_method(),
request.get_full_url(),
request.headers,
data.decode('utf-8') if data else '')
logger.debug(msg)
try:
r = urllib.request.urlopen(request)
return etree.XML(r.read())
except urllib.error.HTTPError as httperr:
logger.error(httperr.read())
raise
except urllib.error.URLError as urlerr:
logger.error(request.get_full_url())
raise
def http_request(self, request):
data = request.get_data()
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if hasattr(value, 'read'):
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
raise TypeError
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if (
request.has_header('Content-Type') and
request.get_header('Content-Type').find(
'multipart/form-data') != 0
):
six.print_(
"Replacing %s with %s" % (
request.get_header('content-type'),
'multipart/form-data'
)
)
request.add_unredirected_header('Content-Type', contenttype)
request.add_data(data)
return request
def http_request(self, request):
data = request.get_data()
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for (key, value) in list(data.items()):
if type(value) == io.BufferedReader:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
# raise TypeError("not a valid non-string sequence or mapping object %d" % traceback)
logger.debug("not a valid non-string sequence or mapping object : %s" % traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, self.doseq).encode()
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = "multipart/form-data; boundary=%s" % boundary
request.add_unredirected_header("Content-Type", contenttype)
request.add_data(data)
return request
def post(self, filename, ccd_data, use_base64=False):
""" filename : The name of the file attachment
ccd_data : The xml to post (send)
use_base64 : If true, encode the encrypted XML as base64. Maybe necessary for 3DES
"""
attachments = ""
payload = """--%s
Content-Disposition: attachment; name="%s"; filename="%s.xml"
Content-Type: text/xml
%s"""
payload_uuid = str(uuid.uuid4())
try:
for data in ccd_data:
if len(attachments) > 0:
attachments += "\r\n\r\n"
if use_base64:
attachments += payload % (payload_uuid, filename, filename, base64.b64encode(data))
else:
attachments += payload % (payload_uuid, filename, filename, data)
#import pdb; pdb.set_trace()
if self._url.find("https") == 0:
https = urllib.request.HTTPSHandler(debuglevel=1)
opener = urllib.request.build_opener(https)
urllib.request.install_opener(opener)
request = urllib.request.Request(self._url)
request.add_header("Content-Type", "multipart/form-data; boundary=%s" % payload_uuid)
request.add_header("User-Agent", "synthesis")
request.add_data(attachments)
if PRINT_HTTP_POST:
print(request.header_items())
print(request.get_data())
print(request.get_method())
print(request.get_host())
return (True, "True")
response = urllib2.urlopen(request).read()
else:
print("**** POSTING TO LOCAL SERVER ****")
request = urllib.request.Request(self._url)
request.add_header("Content-Type", "multipart/form-data; boundary=%s" % payload_uuid)
request.add_header("User-Agent", "synthesis")
request.add_data(attachments)
if PRINT_HTTP_POST:
print(request.header_items())
print(request.get_data())
print(request.get_method())
print(request.get_host())
return (True, "True")
response = urllib2.urlopen(request).read()
# check for some sign of success within the response
if response[0:4] == "202:":
return (True, response)
else:
return (False, response)
except Exception as err:
return (False, "An error occurred while performing an HTTP-POST or receiving the response: (%s)" % str(err))
def _open(self, method, path, headers=None, data=None):
"""Perform an HTTP request.
Args:
method: The HTTP method (GET, PUT, POST, DELETE).
path: The HTTP path to retrieve.
headers: A dictionary of HTTP headers to add.
data: The data to send as the body of the request.
Returns:
A Response object.
"""
url = urllib.parse.urljoin(self.site, path)
self.log.info('%s %s', method, url)
request = self._request(url)
request.set_method(method)
if headers:
for key, value in headers.items():
request.add_header(key, value)
if self.auth:
# Insert basic authentication header
request.add_header('Authorization', 'Basic {}'.format(self.auth.decode()))
if request.headers:
header_string = '\n'.join([':'.join((k, v)) for k, v in
request.headers.items()])
self.log.debug('request-headers:%s', header_string)
if data:
request.add_header('Content-Type', self.format.mime_type)
request.add_data(data)
self.log.debug('request-body:%s', request.get_data())
elif method in ['POST', 'PUT']:
# Some web servers need a content length on all POST/PUT operations
request.add_header('Content-Type', self.format.mime_type)
request.add_header('Content-Length', '0')
if self.timeout and not _urllib_has_timeout():
# Hack around lack of timeout option in python < 2.6
old_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(self.timeout)
try:
http_response = None
try:
http_response = self._handle_error(self._urlopen(request))
except urllib.error.HTTPError as err:
http_response = self._handle_error(err)
except urllib.error.URLError as err:
raise Error(err, url)
response = Response.from_httpresponse(http_response)
self.log.debug('Response(code=%d, headers=%s, msg="%s")',
response.code, response.headers, response.msg)
finally:
if http_response:
http_response.close()
if self.timeout and not _urllib_has_timeout():
socket.setdefaulttimeout(old_timeout)
self.log.info('--> %d %s %db', response.code, response.msg,
len(response.body))
return response
def request(self, host, handler, request_body, verbose):
self.verbose = verbose
url = 'http://{0}{1}'.format(host, handler)
request = urllib.Request(url)
request.add_data(request_body)
request.add_header("User-Agent", self.user_agent)
request.add_header("Content-Type", "text/html")
f = urllib.urlopen(request)
return self.parse_response(f)
def request(self, host, handler, request_body, verbose):
self.verbose = verbose
url = "http://{0}{1}".format(host, handler)
request = urllib.Request(url)
request.add_data(request_body)
request.add_header("User-Agent", self.user_agent)
request.add_header("Content-Type", "text/html")
f = urllib.urlopen(request)
return self.parse_response(f)
def run(self, params=[], auth=True, console=True): params.insert(1, self.auth_token) if params[0] != 'session.list': params.insert(2, self.console_id) #print(params) request = self.set_request() query_params = msgpack.packb(params) request.add_data(query_params) response = msgpack.unpackb(urllib.request.urlopen(request).read()) return response
def post(self, path, headers, body):
uri = "%s/%s" % (self.url.geturl(), path)
if self.debug: print("post: uri=%s, headers=%s" % (uri, headers))
request = urllib2.Request(uri)
for header in headers:
request.add_header(header, headers[header])
request.add_data(body)
resp = urllib2.urlopen(request, cafile=self.cafile)
if self.debug: print("resp: %s" % resp.info())
return resp
def check_app_updates(app_info_list, raw_result=False):
# This function expects a list of dicts with, at a minimum, the 'adam-id' key set
# This ID is the unique product identifier for an app on the App Store and can be found in the store URL
# when viewing the app's page in a web browser, like so:
# https://itunes.apple.com/us/app/evernote/id406056744
# ^^^^^^^^^
# The other 3 keys that will be passed in the search are: CFBundleIdentifier, CFBundleShortVersionString, and
# installed-version-identifier (explained earlier). Lack of any of these keys will result in a filler value
# being provided which, as long as the adam-id is present, the App Store update mechanism doesn't seem to
# care about.
update_url = 'https://su.itunes.apple.com/WebObjects/MZSoftwareUpdate.woa/wa/availableSoftwareUpdatesExtended'
request = urllib.request.Request(update_url)
# Headers #
# This sets us to the US store. See:
# http://blogs.oreilly.com/iphone/2008/08/scraping-appstore-reviews.html
# http://secrets.blacktree.com/edit?id=129761
request.add_header('X-Apple-Store-Front', '143441-1,13')
# This indicates we're sending an XML plist
request.add_header('Content-Type', 'application/x-apple-plist')
# This seems to be the minimum string to be recognized as a valid app store update checker
# Normally, it's of the form: User-Agent: MacAppStore/1.3 (Macintosh; OS X 10.9) AppleWebKit/537.71
request.add_header('User-Agent', 'MacAppStore')
# Build up the plist
local_software = []
for an_app in app_info_list:
app_entry = {
'CFBundleIdentifier':
an_app.get('CFBundleIdentifier', '.'),
'CFBundleShortVersionString':
an_app.get('CFBundleShortVersionString', '0'),
'adam-id':
an_app['adam-id'],
'installed-version-identifier':
an_app.get('installed-version-identifier', 0)
}
local_software.append(app_entry)
plist_dict = {'local-software': local_software}
plist_str = dump_plist(plist_dict)
request.add_data(plist_str)
# Build the connection
response_handle = urllib.request.urlopen(request)
try:
response = response_handle.read()
except HTTPError as e:
raise ProcessorError("Invalid adam-id %s" % e)
response_handle.close()
# Currently returning the raw response
# Initial analysis:
# - It appears that applications that need updating will be under the 'incompatible-items' key
# - 'version-external-identifiers' is a list of historical versions, in order
# - 'version-external-identifier' is the current version
# - 'current-version' is the CFBundleShortVersionString
# - 'bundle-id' is the CFBundleIdentifier
# - 'preflight' is a *very* interesting 'pfpkg'. See details at bottom.
return load_plist(response)
def http_request(self, request):
try:
data = request.get_data()
except AttributeError:
data = request.data
if data is not None and type(data) != str:
data = urllib.parse.urlencode(data, doseq).encode("utf-8")
try:
request.add_data(data)
except AttributeError:
request.data = data
return request
def get_file_data_request(self, url, file_field_name=None, file_data=None, **fields):
msg = MultipartMimeFormData()
msg.add_csv_file(file_field_name, "import.csv", file_data)
for field, value in fields.items():
msg.add_field(field, value)
body = str(msg).encode('utf-8')
request = urllib.request.Request(url)
request.add_header("Content-Type", msg.get_content_type())
request.add_header("Content-Length", len(body))
request.add_header("Referer", self.referer)
request.add_data(body)
return request
def create_console(self):
options = ['console.create']
options.insert(1, self.auth_token)
request = self.set_request()
query_params = msgpack.packb(options)
request.add_data(query_params)
response = msgpack.unpackb(urllib.request.urlopen(request).read())
#print(response)
if response.get(b'id') is None:
print("ERROR!!!!!")
exit()
print("#Console #"+str(response.get(b'id')).replace("b'", "").replace("'","")+" Created")
return response.get(b'id')
def _github_api_request(self, url, data=None, method=None, authenticate=False):
logging.debug("Making github API request {0}".format(url))
request = urllib.request.Request(url)
if method:
request.get_method = lambda: method
if data == "":
# Workaround for PUTs requiring data, even if you have nothing to pass
request.add_data(data)
elif data:
request.add_data(json.dumps(data))
# Manually adding authentication data
# Basic works in curl, but urllib2 doesn't
# probably because github's API doesn't send a www-authenticate header
if authenticate or self._github_have_authorization():
from base64 import encodestring
auth = self._github_authorization()
if ":" in auth:
# username:password
base64string = encodestring(auth).replace('\n', '')
request.add_header("Authorization", "Basic {0}".format(base64string))
else:
# token
request.add_header("Authorization", "Bearer {0}".format(auth))
try:
response = urllib.request.urlopen(request)
except IOError as e:
raise Error("GitHub API failure: " + str(e))
if response.code == 204:
# 204 = No content
return None
json_parsed = json.load(response)
link_headers = response.info().getallmatchingheaders("Link")
if link_headers:
logging.debug("Found a Link header in response, analyzing...")
link_header = link_headers[0].lstrip("Link:")
links_raw = link_header.split(",")
links_split_raw = [link.split(";") for link in links_raw]
links_split_proc = [(l[1].strip().lstrip('rel="').rstrip('"'), l[0].strip().lstrip("<").rstrip(">")) for l in links_split_raw]
links_dict = dict((k, v) for (k, v) in links_split_proc)
if "next" in links_dict:
logging.debug("Link with rel=\"next\" found, recursing to deal with pagination")
rest = self._github_api_request(links_dict["next"], data, method, authenticate)
json_parsed += rest
return json_parsed
def _github_api_request(self, url, data=None, method=None, authenticate=False):
logging.debug("Making github API request {0}".format(url))
request = urllib.request.Request(url)
if method:
request.get_method = lambda: method
if data == "":
# Workaround for PUTs requiring data, even if you have nothing to pass
request.add_data(data)
elif data:
request.add_data(json.dumps(data))
# Manually adding authentication data
# Basic works in curl, but urllib2 doesn't
# probably because github's API doesn't send a www-authenticate header
if authenticate or self._github_have_authorization():
from base64 import encodestring
auth = self._github_authorization()
if ":" in auth:
# username:password
base64string = encodestring(auth).replace('\n', '')
request.add_header("Authorization", "Basic {0}".format(base64string))
else:
# token
request.add_header("Authorization", "Bearer {0}".format(auth))
try:
response = urllib.request.urlopen(request)
except IOError as e:
raise Error("GitHub API failure: " + str(e))
if response.code == 204:
# 204 = No content
return None
json_parsed = json.load(response)
link_headers = response.info().getallmatchingheaders("Link")
if link_headers:
logging.debug("Found a Link header in response, analyzing...")
link_header = link_headers[0].lstrip("Link:")
links_raw = link_header.split(",")
links_split_raw = [link.split(";") for link in links_raw]
links_split_proc = [(l[1].strip().lstrip('rel="').rstrip('"'), l[0].strip().lstrip("<").rstrip(">")) for l in links_split_raw]
links_dict = dict((k, v) for (k, v) in links_split_proc)
if "next" in links_dict:
logging.debug("Link with rel=\"next\" found, recursing to deal with pagination")
rest = self._github_api_request(links_dict["next"], data, method, authenticate)
json_parsed += rest
return json_parsed
def __call__(self, post=False, *args, **kargs):
if len(args):
raise TypeError('Only keyword arguments are allowed')
if type(post) is not bool:
raise TypeError('post argument can only be True or False')
form = _MultiPartForm()
add_form = False
for (k, v) in kargs.items():
if isinstance(v, File):
add_form = True
form.add_file(k, v.get_filename(), v.content)
if post:
url = self._urlbase
for k, v in self._mkarg(kargs).items():
form.add_field(k, v)
add_form = True
else:
url = self.geturl(**kargs)
request = urllib.request.Request(url)
if add_form:
body = str(form)
request.add_header('Content-type', form.get_content_type())
request.add_header('Content-length', str(len(body)))
request.add_data(body)
self._api.update_request(request)
retry = self._api.max_retries
while True:
retry -= 1
try:
ret = urllib.request.urlopen(request,
timeout=self._api.timeout).read()
break
except urllib.error.HTTPError as e:
raise APIError(e.code, url, e.read())
except (socket.error, urllib.error.URLError) as e:
if retry < 0:
raise e
_print_debug('caught error: {}; retrying'.format(e))
time.sleep(self._api.retry_delay)
if self._api.decode_result:
try:
ret = json.loads(ret.decode())
except:
raise APIError(-1, url,
'json decode error, value={0!r}'.format(ret))
return ret
def fetch(self, url, debug=False, data=None):
"""
Fetches a web page.
If data is None, a GET request will be done, else it will be a POST request using data (must be encoded using urllib.parse.urlencode).
"""
u = None
opener = None
request = None
try:
# Building opener
if self.proxy != None:
if debug:
print(("Using proxy %s" % self.proxy))
proxy = urllib.request.ProxyHandler({'http': self.proxy})
opener = urllib.request.build_opener(proxy)
#urllib2.install_opener(opener)
#urllib2.urlopen('http://www.google.com')
else:
opener = urllib.request.build_opener()
# Building request
request = urllib.request.Request(url)
request.add_header(
"User-Agent",
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.6) Gecko/20050512 Firefox"
)
request.add_data(data)
# Opening URL
u = opener.open(request)
except Exception as e:
if debug:
print(("Couldn't fetch %s" % url))
print(e)
self.notfound += 1
self.current_url = None
return None
try:
l = u.read()
self.current_url = u.geturl()
except Exception as e:
if debug:
print("Couldn't read data from socket")
print(e)
self.notfound += 1
self.current_url = None
return None
return l
def __call__(self, url, params):
data = json.dumps({
"jsonrpc": "2.0",
"method": "call",
"params": params,
"id": random.randint(0, 1000000000),
})
request = urllib.request.Request(url='/'.join([self._root_url, url]))
request.add_header('Content-Type', 'application/json')
request.add_data(data)
response = self._opener.open(request, timeout=self._timeout)
if not self._deserialize:
return response
return json.load(response)
def __call__(self, post = False, *args, **kargs):
if len(args):
raise TypeError('Only keyword arguments are allowed')
if type(post) is not bool:
raise TypeError('post argument can only be True or False')
form = _MultiPartForm()
add_form = False
for (k, v) in kargs.items():
if isinstance(v, File):
add_form = True
form.add_file(k, v.get_filename(), v.content)
if post:
url = self._urlbase
for k, v in self._mkarg(kargs).items():
form.add_field(k, v)
add_form = True
else:
url = self.geturl(**kargs)
request = urllib.request.Request(url)
if add_form:
body = str(form)
request.add_header('Content-type', form.get_content_type())
request.add_header('Content-length', str(len(body)))
request.add_data(body)
self._api.update_request(request)
retry = self._api.max_retries
while True:
retry -= 1
try:
ret = urllib.request.urlopen(request, timeout = self._api.timeout).read()
break
except urllib.error.HTTPError as e:
raise APIError(e.code, url, e.read())
except (socket.error, urllib.error.URLError) as e:
if retry < 0:
raise e
_print_debug('caught error: {}; retrying'.format(e))
time.sleep(self._api.retry_delay)
if self._api.decode_result:
try:
ret = json.loads(ret.decode())
except:
raise APIError(-1, url, 'json decode error, value={0!r}'.format(ret))
return ret
def mylogin(self):
options = ['auth.login', self.user, self.password]
token = None
request = self.set_request()
query_params = msgpack.packb(options)
request.add_data(query_params)
response = msgpack.unpackb(urllib.request.urlopen(request).read())
#print(response)
if response.get(b'result') == b'success':
print("#Authed")
token = response.get(b'token')
else:
print("#GTFO!!!!!!")
exit()
return token
def marshall(self, args, files, request):
"""
Marshalls the arguments to the CGI script as multi-part/form-data,
not the default application/x-www-form-url-encoded. This improves
the transfer of the large inputs and eases command line invocation
of the CGI script.
"""
(contentType, body) = self.encode(args, files)
request.add_header(b'Content-Type', contentType)
if PY2:
request.add_header('Content-Length', str(len(body)))
request.add_data(body)
elif PY3:
request.data = body
return
def __send_request(self, method, uri, data):
"""Summary.
Args:
method (TYPE): Description
uri (TYPE): Description
data (TYPE): Description
Returns:
TYPE: Description
Raises:
APIError: Description
"""
url = self.__url + uri
request = urllib.request.Request(url)
if (method == 'POST'):
request.add_data(json.dumps(data))
auth = base64.b64encode('%s:%s' % (self.user, self.password))
request.add_header('Authorization', 'Basic %s' % auth)
request.add_header('Content-Type', 'application/json')
if self.debug:
handler = urllib.request.HTTPHandler(debuglevel=1)
opener = urllib.request.build_opener(handler)
urllib.request.install_opener(opener)
e = None
try:
response = urllib.request.urlopen(request).read()
except urllib.error.HTTPError as e:
response = e.read()
if response:
result = json.loads(response)
else:
result = {}
if e is not None:
if result and 'error' in result:
error = '"' + result['error'] + '"'
else:
error = 'No additional error message received'
raise APIError('TestRail API returned HTTP %s (%s)' %
(e.code, error))
if isinstance(result, list):
return [AttrDict(x) for x in result]
return AttrDict(result)
def get_file_data_request(self,
url,
file_field_name=None,
file_data=None,
**fields):
msg = MultipartMimeFormData()
msg.add_csv_file(file_field_name, "import.csv", file_data)
for field, value in fields.items():
msg.add_field(field, value)
body = str(msg).encode('utf-8')
request = urllib.request.Request(url)
request.add_header("Content-Type", msg.get_content_type())
request.add_header("Content-Length", len(body))
request.add_header("Referer", self.referer)
request.add_data(body)
return request
def request(self, host, handler, request_body, verbose):
self.verbose = verbose
url = self.schema + "://" + host + handler
request = urllib.request.Request(url)
request.add_data(request_body)
# Note: 'Host' and 'Content-Length' are added automatically
request.add_header("User-Agent", self.user_agent)
request.add_header("Content-Type", "text/xml") # Important
proxy_handler = urllib.request.ProxyHandler()
opener = urllib.request.build_opener(proxy_handler)
f = opener.open(request)
return self.parse_response(f)
def fetch(self, url, debug=False, data=None):
"""
Fetches a web page.
If data is None, a GET request will be done, else it will be a POST request using data (must be encoded using urllib.parse.urlencode).
"""
u = None
opener = None
request = None
try:
# Building opener
if self.proxy != None:
if debug:
print(("Using proxy %s" % self.proxy))
proxy= urllib.request.ProxyHandler({'http': self.proxy})
opener = urllib.request.build_opener(proxy)
#urllib2.install_opener(opener)
#urllib2.urlopen('http://www.google.com')
else:
opener = urllib.request.build_opener()
# Building request
request = urllib.request.Request(url)
request.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.6) Gecko/20050512 Firefox")
request.add_data(data)
# Opening URL
u = opener.open(request)
except Exception as e:
if debug:
print(("Couldn't fetch %s" % url))
print(e)
self.notfound += 1
self.current_url = None
return None
try:
l = u.read()
self.current_url = u.geturl()
except Exception as e:
if debug:
print("Couldn't read data from socket")
print(e)
self.notfound += 1
self.current_url = None
return None
return l
def _action(self, params, body=None, content_type=None):
# about token, see https://github.com/bittorrent/webui/wiki/TokenSystem
url = (
self.base_url + '?token=' +
self.token + '&' + urllib.parse.urlencode(params)
)
request = urllib.request.Request(url)
if body:
request.add_data(body)
request.add_header('Content-length', len(body))
if content_type:
request.add_header('Content-type', content_type)
try:
response = self.opener.open(request)
return response.code, json.loads(response.read().decode('utf-8'))
except urllib.error.HTTPError:
raise
def get_trusted_ticket_for_user(self, username, site='default', ip=None):
trusted_url = self.tableau_server_url + "/trusted"
opener = urllib.request.build_opener(urllib.request.HTTPHandler)
request = urllib.request.Request(trusted_url)
post_data = "username={}".format(username)
if site.lower() != 'default':
post_data += "&target_site={}".format(site)
request.add_data(post_data)
trusted_ticket_response = opener.open(request)
try:
ticket = trusted_ticket_response.read()
if ticket == '-1' or not ticket:
raise NoResultsException('Ticket generation was not complete.')
else:
return ticket
except urllib.error.HTTPError as e:
if e.code >= 500:
raise
raw_error_response = e.fp.read()
def MultipartPost(url, fields):
"""
Post form data specified by fields (a dictionary) to the specified url.
Returns a urllib2-style file object.
"""
content_type, body = encode_multipart_formdata(fields)
request = Request(url)
request.add_header('User-Agent', BROWSER_MASQUERADE)
request.add_header('Content-Type', content_type)
request.add_header('Content-Length', str(len(body)))
if usingPython2:
request.add_data(body)
else:
# Python 3: add_data method is deprecated; body must be
# converted from Python 3 string to bytes for Request object
request.data = body.encode('utf-8')
connection = urlopen(request)
return connection
def sendSpeechData(self, vid, data):
url = self._baseAsrUrl + 'voices/' + self._uuid
headers = {
'Content-Type': 'multipart/form-data',
'X-Token': self._token
}
form_data = ""
form_data += self._boundary + "\r\n"
form_data += "Content-Disposition: form-data;name=\"voice_id\"\r\n\r\n"
form_data += str(vid) + "\r\n"
form_data += self._boundary + "\r\n"
form_data += "Content-Disposition: form-data;name=\"voice\"\r\n"
form_data += "Content-Type: application/octet-stream\r\n\r\n"
form_data += data
form_data += "\r\n"
form_data += self._boundary + "\r\n"
request = urllib.request.Request(url)
request.add_header('X-Token', self._token)
request.add_header('Content-Type', 'multipart/form-data')
request.add_data(bytearray(form_data))
request.get_method = lambda: 'PUT'
try:
result = urllib.request.urlopen(request)
except urllib.error.HTTPError as e:
print('Error code:', e.code)
print('Reason:', e.reason)
return False
except urllib.error.URLError as e:
print('URLErroe reason:', e.reason)
return False
else:
response = result.read()
res = response.decode('utf-8')
if res:
return res
return False
def getUrlFd(url, headers=None, data=None, timeout=None):
"""getUrlFd(url, headers=None, data=None, timeout=None)
Opens the given url and returns a file object. Headers and data are
a dict and string, respectively, as per urllib.request.Request's
arguments."""
if headers is None:
headers = defaultHeaders
if minisix.PY3 and isinstance(data, str):
data = data.encode()
try:
if not isinstance(url, Request):
(scheme, loc, path, query, frag) = urlsplit(url)
(user, host) = splituser(loc)
url = urlunsplit((scheme, host, path, query, ''))
request = Request(url, headers=headers, data=data)
if user:
request.add_header('Authorization',
'Basic %s' % base64.b64encode(user))
else:
request = url
request.add_data(data)
httpProxy = force(proxy)
if httpProxy:
request.set_proxy(httpProxy, 'http')
fd = urlopen(request, timeout=timeout)
return fd
except socket.timeout as e:
raise Error(TIMED_OUT)
except sockerrors as e:
raise Error(strError(e))
except InvalidURL as e:
raise Error('Invalid URL: %s' % e)
except HTTPError as e:
raise Error(strError(e))
except URLError as e:
raise Error(strError(e.reason))
# Raised when urllib doesn't recognize the url type
except ValueError as e:
raise Error(strError(e))
def getUrlFd(url, headers=None, data=None, timeout=None):
"""getUrlFd(url, headers=None, data=None, timeout=None)
Opens the given url and returns a file object. Headers and data are
a dict and string, respectively, as per urllib.request.Request's
arguments."""
if headers is None:
headers = defaultHeaders
if minisix.PY3 and isinstance(data, str):
data = data.encode()
try:
if not isinstance(url, Request):
(scheme, loc, path, query, frag) = urlsplit(url)
(user, host) = splituser(loc)
url = urlunsplit((scheme, host, path, query, ''))
request = Request(url, headers=headers, data=data)
if user:
request.add_header('Authorization',
'Basic %s' % base64.b64encode(user))
else:
request = url
request.add_data(data)
httpProxy = force(proxy)
if httpProxy:
request.set_proxy(httpProxy, 'http')
fd = urlopen(request, timeout=timeout)
return fd
except socket.timeout as e:
raise Error(TIMED_OUT)
except sockerrors as e:
raise Error(strError(e))
except InvalidURL as e:
raise Error('Invalid URL: %s' % e)
except HTTPError as e:
raise Error(strError(e))
except URLError as e:
raise Error(strError(e.reason))
# Raised when urllib doesn't recognize the url type
except ValueError as e:
raise Error(strError(e))
def http_request(self, request):
data = request.get_data()
def isfiledata(p_str):
import re
r_c = re.compile("^f'(.*)'$")
rert = r_c.search(str(p_str))
#rert = re.search("^f'(.*)'$", p_str)
if rert:
return rert.group(1)
else:
return None
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if isfiledata(value): # type(value) == file:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if(request.has_header('Content-Type')
and request.get_header('Content-Type').find('multipart/form-data') != 0):
print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data'))
request.add_unredirected_header('Content-Type', contenttype)
request.add_data(data)
return request
urlPost = 'https://api.medium.com/v1/users/%s/posts' % authorId
values = dict()
values['title'] = "Hola Medium!"
values['contentFormath'] = "html"
values[
'content'] = "<h1>Hola Medium!</h1><p>Probando el API de Medium desde Pythonista.</p>"
values['publishStatus'] = "draft"
data = json.dumps(values)
request = urllib.request.Request(urlPost)
request.add_header("Authorization", "Bearer %s" % integrationToken)
request.add_header("Content-Type", contentTypeHeader)
request.add_header("Accept", acceptHeader)
request.add_header("Accept-Charset", acceptCharsetHeader)
request.add_data(data)
try:
response = urllib.request.urlopen(request)
except urllib.error.HTTPError as e:
if e.code == 400:
print("Incorrect fields. Bad request")
print(e.reason)
elif e.code == 401:
print("The integration token is invalid")
print(e.reason)
elif e.code == 403:
print("User without permission to publish.")
print(e.reason)
elif e.code == 404:
print("User unknown")
print(e.reason)
def upload(self, filename=None, jpegData=None, **arg):
"""Upload a file to flickr.
Be extra careful you spell the parameters correctly, or you will
get a rather cryptic "Invalid Signature" error on the upload!
Supported parameters:
One of filename or jpegData must be specified by name when
calling this method:
filename -- name of a file to upload
jpegData -- array of jpeg data to upload
api_key
auth_token
title
description
tags -- space-delimited list of tags, "tag1 tag2 tag3"
is_public -- "1" or "0"
is_friend -- "1" or "0"
is_family -- "1" or "0"
"""
if filename == None and jpegData == None or \
filename != None and jpegData != None:
raise UploadException("filename OR jpegData must be specified")
# verify key names
for a in list(arg.keys()):
if a != "api_key" and a != "auth_token" and a != "title" and \
a != "description" and a != "tags" and a != "is_public" and \
a != "is_friend" and a != "is_family":
sys.stderr.write("FlickrAPI: warning: unknown parameter " \
"\"%s\" sent to FlickrAPI.upload\n" % (a))
arg["api_sig"] = self.__sign(arg)
url = "http://" + FlickrAPI.flickrHost + FlickrAPI.flickrUploadForm
# construct POST data
boundary = mimetools.choose_boundary()
body = ""
# required params
for a in ('api_key', 'auth_token', 'api_sig'):
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\""+a+"\"\r\n\r\n"
body += "%s\r\n" % (arg[a])
# optional params
for a in ('title', 'description', 'tags', 'is_public', \
'is_friend', 'is_family'):
if a in arg:
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\""+a+"\"\r\n\r\n"
body += "%s\r\n" % (arg[a])
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\"photo\";"
body += " filename=\"%s\"\r\n" % filename
body += "Content-Type: image/jpeg\r\n\r\n"
#print body
if filename != None:
fp = file(filename, "rb")
data = fp.read()
fp.close()
else:
data = jpegData
postData = body.encode("utf_8") + data + \
("--%s--" % (boundary)).encode("utf_8")
request = urllib.request.Request(url)
request.add_data(postData)
request.add_header("Content-Type", \
"multipart/form-data; boundary=%s" % boundary)
response = urllib.request.urlopen(request)
rspXML = response.read()
return XMLNode.parseXML(rspXML)
def http_request(
url,
json_string,
username = None,
password = None,
timeout = None,
additional_headers = None,
content_type = None,
cookies = None,
gzipped = None,
ssl_context = None,
debug = None
):
"""
Fetch data from webserver (POST request)
:param json_string: JSON-String
:param username: If *username* is given, BASE authentication will be used.
:param timeout: Specifies a timeout in seconds for blocking operations
like the connection attempt (if not specified, the global default
timeout setting will be used).
See: https://github.com/gerold-penz/python-jsonrpc/pull/6
:param additional_headers: Dictionary with additional headers
See: https://github.com/gerold-penz/python-jsonrpc/issues/5
:param content_type: Possibility to change the content-type header.
:param cookies: Possibility to add simple cookie-items as key-value pairs.
The key and the value of each cookie-item must be a bytestring.
Unicode is not allowed here.
:param gzipped: If `True`, the JSON-String will be gzip-compressed.
:param ssl_context: Specifies custom TLS/SSL settings for connection.
Python > 2.7.9
See: https://docs.python.org/2/library/ssl.html#client-side-operation
:param debug: If `True` --> *logging.debug*
"""
# Debug
if debug:
logging.debug("Client-->Server: {json_string}".format(json_string=repr(json_string)))
# Create request and add data
request = urllib2.Request(url)
if gzipped:
# Compress content (SpooledTemporaryFile to reduce memory usage)
spooled_file = tools.SpooledFile()
tools.gzip_str_to_file(json_string, spooled_file)
del json_string
request.add_header("Content-Encoding", "gzip")
request.add_header("Accept-Encoding", "gzip")
spooled_file.seek(0)
if six.PY2:
request.add_data(spooled_file)
else:
request.data = spooled_file
else:
if six.PY2:
request.add_data(json_string)
else:
request.data = json_string
# Content Type
request.add_header("Content-Type", content_type or "application/json")
# Authorization
if username:
base64string = base64.b64encode("%s:%s" % (username, password)).strip()
request.add_unredirected_header("Authorization", "Basic %s" % base64string)
# Cookies
if cookies:
cookie = Cookie.SimpleCookie(cookies)
request.add_header("Cookie", cookie.output(header = "", sep = ";"))
# Additional headers (overrides other headers)
if additional_headers:
for key, val in six.iteritems(additional_headers):
request.add_header(key, val)
# Send request to server
http_error_exception = urllib2.HTTPError if six.PY2 else urllib.error.HTTPError
try:
if ssl_context:
try:
response = urllib2.urlopen(
request, timeout=timeout, context=ssl_context
)
except TypeError as err:
if "context" in unicode(err):
raise NotImplementedError("SSL-Context needs Python >= 2.7.9")
else:
raise
else:
response = urllib2.urlopen(request, timeout=timeout)
except http_error_exception as err:
if debug:
retval = err.read()
logging.debug("Client<--Server: {retval}".format(retval=repr(retval)))
raise
# Analyze response and return result
try:
if "gzip" in response.headers.get("Content-Encoding", ""):
response_file = tools.SpooledFile(source_file = response)
if debug:
retval = tools.gunzip_file(response_file)
logging.debug("Client<--Server: {retval}".format(retval = repr(retval)))
return retval
return tools.gunzip_file(response_file)
else:
if debug:
retval = response.read()
logging.debug("Client<--Server: {retval}".format(retval=repr(retval)))
return retval
return response.read()
finally:
response.close()
def add_to_request(self, request):
body = str(self).encode('utf-8')
request.add_header('Content-type', self.get_content_type())
request.add_header('Content-length', str(len(body)))
request.add_data(body)
def upload(self, filename=None, jpegData=None, **arg):
"""Upload a file to flickr.
Be extra careful you spell the parameters correctly, or you will
get a rather cryptic "Invalid Signature" error on the upload!
Supported parameters:
One of filename or jpegData must be specified by name when
calling this method:
filename -- name of a file to upload
jpegData -- array of jpeg data to upload
api_key
auth_token
title
description
tags -- space-delimited list of tags, "tag1 tag2 tag3"
is_public -- "1" or "0"
is_friend -- "1" or "0"
is_family -- "1" or "0"
"""
if filename == None and jpegData == None or \
filename != None and jpegData != None:
raise UploadException("filename OR jpegData must be specified")
# verify key names
for a in list(arg.keys()):
if a != "api_key" and a != "auth_token" and a != "title" and \
a != "description" and a != "tags" and a != "is_public" and \
a != "is_friend" and a != "is_family":
sys.stderr.write("FlickrAPI: warning: unknown parameter " \
"\"%s\" sent to FlickrAPI.upload\n" % (a))
arg["api_sig"] = self.__sign(arg)
url = "https://" + FlickrAPI.flickrHost + FlickrAPI.flickrUploadForm
# construct POST data
boundary = email.generator._make_boundary()
body = ""
# required params
for a in ('api_key', 'auth_token', 'api_sig'):
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\"" + a + "\"\r\n\r\n"
body += "%s\r\n" % (arg[a])
# optional params
for a in ('title', 'description', 'tags', 'is_public', \
'is_friend', 'is_family'):
if a in arg:
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\"" + a + "\"\r\n\r\n"
body += "%s\r\n" % (arg[a])
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\"photo\";"
body += " filename=\"%s\"\r\n" % filename
body += "Content-Type: image/jpeg\r\n\r\n"
#print body
if filename != None:
fp = file(filename, "rb")
data = fp.read()
fp.close()
else:
data = jpegData
postData = body.encode("utf_8") + data + \
("--%s--" % (boundary)).encode("utf_8")
request = urllib.request.Request(url)
request.add_data(postData)
request.add_header("Content-Type", \
"multipart/form-data; boundary=%s" % boundary)
response = urllib.request.urlopen(request)
rspXML = response.read()
return XMLNode.parseXML(rspXML)
base64_consumer_key_secret = base64.b64encode(
urllib.parse.quote(CONSUMER_KEY) + b':' +
urllib.parse.quote(CONSUMER_SECRET))
#
# Step 2: Obtain a bearer token
#
# note: the following line won't verify server certificate; to do so you'll have to
# use python3 and specify cafile & capauth
request = urllib.request.Request("https://api.twitter.com/oauth2/token")
request.add_header('Authorization', b'Basic ' + base64_consumer_key_secret)
request.add_header("Content-Type",
b'application/x-www-form-urlencoded;charset=UTF-8')
request.add_data(b'grant_type=client_credentials')
resp = urllib.request.urlopen(request)
data = json.load(resp)
if data['token_type'] != 'bearer':
throw("Bad token_type: " + data['token_type'])
access_token = data['access_token']
print(("access_token: " + access_token))
print('')
#
# Step 3: Authenticate API requests with the bearer token
#
request = urllib.request.Request(
#!/usr/env/bin python3
# -*- coding:utf-8 -*-
import urllib.request, urllib.error, http.cookiejar
# 创建cookie容器
cj = http.cookiejar.CookieJar()
# 创建一个opener
opener = urllib.request.build_opener(urllib2.HTTPCookieProcessor(cj))
# 给urllib2安装opener
urllib.request.install_opener(opener)
# 创建Request对象
request = urllib.request.Request(url)
# 添加数据
request.add_data('a', '1')
# 添加http的header
request.add_header('User-Agent', 'Mozilla/5.0')
# 发送请求获取结果
response = urllib.request.urlopen(request)
def plugin(srv, item):
""" addrs: (method, url, dict(params), list(username, password), json) """
srv.logging.debug("*** MODULE=%s: service=%s, target=%s", __file__,
item.service, item.target)
method = item.addrs[0]
url = item.addrs[1]
params = item.addrs[2]
timeout = item.config.get('timeout', 60)
auth = None
try:
username, password = item.addrs[3]
auth = base64.encodestring('%s:%s' % (username, password)).replace(
'\n', '')
except:
pass
tojson = None
try:
tojson = item.addrs[4]
except:
pass
# Try and transform the URL. Use original URL if it's not possible
try:
url = url.format(**item.data)
except:
pass
if params is not None:
for key in list(params.keys()):
# { 'q' : '@message' }
# Quoted field, starts with '@'. Do not use .format, instead grab
# the item's [message] and inject as parameter value.
if params[key].startswith('@'): # "@message"
params[key] = item.get(params[key][1:], "NOP")
else:
try:
params[key] = params[key].format(
**item.data).encode('utf-8')
except Exception as e:
srv.logging.debug("Parameter %s cannot be formatted: %s" %
(key, e))
return False
message = item.message
if method.upper() == 'GET':
try:
if params is not None:
resource = url
if not resource.endswith('?'):
resource = resource + '?'
resource = resource + urllib.parse.urlencode(params)
else:
resource = url
request = urllib.request.Request(resource)
request.add_header('User-agent', srv.SCRIPTNAME)
if auth is not None:
request.add_header("Authorization", "Basic %s" % auth)
resp = urllib.request.urlopen(request, timeout=timeout)
data = resp.read()
except Exception as e:
srv.logging.warn("Cannot GET %s: %s" % (resource, e))
return False
return True
if method.upper() == 'POST':
try:
request = urllib.request.Request(url)
if params is not None:
if tojson is not None:
encoded_params = json.dumps(params)
request.add_header('Content-Type', 'application/json')
else:
encoded_params = urllib.parse.urlencode(params)
else:
encoded_params = message
request.add_data(encoded_params)
request.add_header('User-agent', srv.SCRIPTNAME)
if auth is not None:
request.add_header("Authorization", "Basic %s" % auth)
resp = urllib.request.urlopen(request, timeout=timeout)
data = resp.read()
# print "POST returns ", data
except Exception as e:
srv.logging.warn("Cannot POST %s: %s" % (url, e))
return False
return True
srv.logging.warn("Unsupported HTTP method: %s" % (method))
return False
return '\r\n'.join(flattened)
if __name__ == '__main__':
# Create the form with simple fields
form = MultiPartForm()
form.add_field('firstname', 'Doug')
form.add_field('lastname', 'Hellmann')
# Add a fake file
form.add_file('biography',
'bio.txt',
fileHandle=StringIO('Python developer and blogger.'))
# Build the request
request = urllib.request.Request('http://localhost:8080/')
request.add_header('User-agent',
'PyMOTW (http://www.doughellmann.com/PyMOTW/)')
body = str(form)
request.add_header('Content-type', form.get_content_type())
request.add_header('Content-length', len(body))
request.add_data(body)
print()
print('OUTGOING DATA:')
print(request.get_data())
print()
print('SERVER RESPONSE:')
print(urllib.request.urlopen(request).read())
