def post_url(self, url, data=None, upload_file=None, upload_file_length=None,
content_type='application/x-www-form-urlencoded',
headers = None, **kwargs):
"""Performs an HTTP POST using pycurl. If ``headers`` is provided, it
will have Content-Type and Content-Length added to it. See
:meth:`_common_perform` for further details.
:param data: The data to use as the POST body. Will over-ride\
``upload_file`` and ``upload_file_length`` if provided.
:type data: str or unicode
:param upload_file: The data to use as the POST body.
:type upload_file: ``.read()``-able file-like object
:param upload_file_length: The length of ``upload_file``. If\
``upload_file`` is provided and this is not, ``friendly_curl`` will use\
``os.fstat`` to calculate it.
:param content_type: The type of the data being POSTed."""
headers = headers or {}
self.curl_handle.setopt(pycurl.POST, 1)
if data:
upload_file = StringIO(data)
upload_file_length = len(data)
if not upload_file_length and hasattr(upload_file, 'fileno'):
upload_file_length = os.fstat(upload_file.fileno()).st_size
self.curl_handle.setopt(pycurl.READFUNCTION, upload_file.read)
headers['Content-Type'] = content_type
headers['Content-Length'] = upload_file_length
try:
result = self._common_perform(url, headers, **kwargs)
finally:
self.reset()
return result
def putf(f, name):
"""Similar to put.
putf, however, accepts a file object (file, StringIO, etc.) ``f`` instead of a file_path.
.. note::
``f`` is not rewound. f.read() from current position will be placed on PiCloud
.. warning::
If the file object does not correspond to an actual file on disk,
it will be read entirely into memory before being transferred to PiCloud."""
if '../..' in name:
raise ValueError('"../.." cannot be in name')
fsize = 0 # file size. may not be computable
if isinstance(f, basestring):
fsize = len(f)
from cStringIO import StringIO
f = StringIO(f)
else:
try:
fsize = os.fstat(f.fileno()).st_size
except (AttributeError, OSError):
pass
if fsize > 5000000000:
raise ValueError('Cannot store files larger than 5GB on cloud.files')
conn = _getcloudnetconnection()
try:
# get a file ticket
resp = conn.send_request(_file_new_query, {'name': name})
ticket = resp['ticket']
params = resp['params']
url = params['action']
# set file in ticket
ticket['file'] = f
# post file using information in ticket
ticket['key'] = str(ticket['key'])
resp = _aws_retryable_post(conn, url, ticket)
resp.read()
finally:
f.close()
def open_data(self, url, data=None):
"""Use "data" URL."""
# ignore POSTed data
#
# syntax of data URLs:
# dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
# mediatype := [ type "/" subtype ] *( ";" parameter )
# data := *urlchar
# parameter := attribute "=" value
import mimetools
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
try:
[type, data] = url.split(',', 1)
except ValueError:
raise IOError, ('data error', 'bad data URL')
if not type:
type = 'text/plain;charset=US-ASCII'
semi = type.rfind(';')
if semi >= 0 and '=' not in type[semi:]:
encoding = type[semi + 1:]
type = type[:semi]
else:
encoding = ''
msg = []
msg.append(
'Date: %s' %
time.strftime('%a, %d %b %Y %T GMT', time.gmtime(time.time())))
msg.append('Content-type: %s' % type)
if encoding == 'base64':
import base64
data = base64.decodestring(data)
else:
data = unquote(data)
msg.append('Content-length: %d' % len(data))
msg.append('')
msg.append(data)
msg = '\n'.join(msg)
f = StringIO(msg)
headers = mimetools.Message(f, 0)
f.fileno = None # needed for addinfourl
return addinfourl(f, headers, url)
def open_data(self, url, data=None):
"""Use "data" URL."""
# ignore POSTed data
#
# syntax of data URLs:
# dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
# mediatype := [ type "/" subtype ] *( ";" parameter )
# data := *urlchar
# parameter := attribute "=" value
import mimetools
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
try:
[type, data] = url.split(',', 1)
except ValueError:
raise IOError, ('data error', 'bad data URL')
if not type:
type = 'text/plain;charset=US-ASCII'
semi = type.rfind(';')
if semi >= 0 and '=' not in type[semi:]:
encoding = type[semi+1:]
type = type[:semi]
else:
encoding = ''
msg = []
msg.append('Date: %s'%time.strftime('%a, %d %b %Y %T GMT',
time.gmtime(time.time())))
msg.append('Content-type: %s' % type)
if encoding == 'base64':
import base64
data = base64.decodestring(data)
else:
data = unquote(data)
msg.append('Content-length: %d' % len(data))
msg.append('')
msg.append(data)
msg = '\n'.join(msg)
f = StringIO(msg)
headers = mimetools.Message(f, 0)
f.fileno = None # needed for addinfourl
return addinfourl(f, headers, url)
def put_url(self, url, data=None, upload_file=None, upload_file_length=None,
content_type='application/x-www-form-urlencoded',
headers = None, **kwargs):
"""Perform an HTTP PUT using pycurl. See :meth:`post_url` and
:meth:`_common_perform` for further details."""
headers = headers or {}
self.curl_handle.setopt(pycurl.UPLOAD, 1)
if data:
upload_file = StringIO(data)
upload_file_length = len(data)
if not upload_file_length and hasattr(upload_file, 'fileno'):
upload_file_length = os.fstat(upload_file.fileno()).st_size
self.curl_handle.setopt(pycurl.READFUNCTION, upload_file.read)
headers['Content-Type'] = content_type
headers['Content-Length'] = upload_file_length
headers['Transfer-Encoding'] = ''
try:
result = self._common_perform(url, headers, **kwargs)
finally:
self.reset()
return result
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
print 'trans start ', self.path
if -1 != self.path.find('//'):
pos = self.path.find('/') + 2
pos += self.path[pos:].find('/')
self.path = self.path[pos:]
print self.path
path = self.translate_path(self.path)
print 'trans end ', path
f = None
query = self.path.find('?')
if query != -1:
if self.get_element(self.path, 'app_id') == APP_ID:
print 'matched'
else:
self.send_error(404, "appid error")
return None
if self.get_element(self.path, 'ver') == APP_VER:
self.send_error(404, "no update")
else:
f = StringIO()
if f:
f.write(download_url_for_app)
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("connection", "keep-alive")
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
if os.path.isdir(path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def send_head(self):
self.path = self.path.replace('..', '')
path = self.translate_path(self.path)
f = None
# print self.path
if self.path.endswith('~editor'):
editortmplfile = os.path.join(libdir, 'editor.html')
if not EditOnlineRequestHandler.editortmpl or True:
try:
EditOnlineRequestHandler.editortmpl = ''.join(open(editortmplfile).readlines())
except IOError:
self.send_error(500, "The editor template file(%s) not found. Maybe need reinstall EditOnline" % editortmplfile)
return None
cxt = {
'path':self.path,
'version':__version__,
'realm':options.get('realm') or '',
'exists':'true' if os.path.exists(self.translate_path(self.path.replace('~editor', ''))) else 'false',
}
res = EditOnlineRequestHandler.editortmpl
for k, v in cxt.items():
res = res.replace('{{%s}}' % k, v)
f = StringIO()
f.write(res)
self.send_response(200)
self.send_header("Content-type", 'text/html')
self.send_header("Content-Length", str(f.tell()))
self.send_header('Connection', 'close')
self.end_headers()
f.seek(0)
return f
elif os.path.isdir(path):
parts = urlparse.urlsplit(self.path)
if not parts.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
new_parts = (parts[0], parts[1], parts[2] + '/',
parts[3], parts[4])
new_url = urlparse.urlunsplit(new_parts)
self.send_header("Location", new_url)
self.send_header('Connection', 'close')
self.end_headers()
return None
else:
return self.list_directory(path)
else:
path = self.path
if path.startswith('/eo.static/'):
# static
path = os.path.join(libdir, path.replace('/eo.static/', 'static/'))
else:
path = self.translate_path(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.send_header('Connection', 'close')
self.end_headers()
return f
except:
f.close()
raise
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
# print path
f = None
codePath=os.getcwd()+"/bin/slave"
moudlePath=os.getcwd()+"/bin/domainRecorder"
userAgent=self.headers.get('User-Agent')
if("Python-urllib/2.7" not in userAgent):
f = StringIO()
f.write('''<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Spider使用说明(wonderkun)</title>
</head>
<body>
关于Spider的使用说明,仅仅需要两步,就可以了</br>
一.在服务器端运行main.y</br>
二.在客户端运行 python -c "exec(__import__('urllib2').urlopen('http://127.0.0.1:8000/').read())",就可以了</br>
三.然后就没有然后了</br>
</body>
</html>''')
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
return f
#输出index.html
elif (path!=codePath) and (path!=moudlePath):
try:
f=open('code.py','rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
fs = os.fstat(f.fileno())
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "gzip; charset=%s" % encoding)
self.send_header("Content-Length",str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except :
f.close()
raise
elif path==codePath:
#show codeObject
try:
# print codePath
f=open(codePath,'rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
fs = os.fstat(f.fileno())
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "gzip; charset=%s" % encoding)
self.send_header("Content-Length",str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except :
f.close()
raise
elif path==moudlePath:
#show codeObject
try:
# print codePath
f=open(moudlePath,'rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
fs = os.fstat(f.fileno())
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "gzip; charset=%s" % encoding)
self.send_header("Content-Length",str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except :
f.close()
raise
class TeeInput(object):
CHUNK_SIZE = sock.CHUNK_SIZE
def __init__(self, response, release_connection=None):
self.buf = StringIO()
self.response = response
self.release_connection = release_connection
self._len = None
self._clen = 0
self.eof = False
# set temporary body
if isinstance(response.body.reader, LengthReader):
self._len = response.body.reader.length
if (response.body.reader.length <= sock.MAX_BODY):
self.tmp = StringIO()
else:
self.tmp = tempfile.TemporaryFile()
else:
self.tmp = tempfile.TemporaryFile()
@property
def len(self):
if self._len is not None:
return self._len
if not self.eof:
pos = self.tmp.tell()
self.tmp.seek(0, 2)
while True:
if not self._tee(self.CHUNK_SIZE):
break
self.tmp.seek(pos)
self._len = self._tmp_size()
return self._len
__len__ = len
def seek(self, offset, whence=0):
""" naive implementation of seek """
current_size = self._tmp_size()
diff = 0
if whence == 0:
diff = offset - current_size
elif whence == 2:
diff = (self.tmp.tell() + offset) - current_size
elif whence == 3 and not self.eof:
# we read until the end
while True:
self.tmp.seek(0, 2)
if not self._tee(self.CHUNK_SIZE):
break
if not self.eof and diff > 0:
self._ensure_length(StringIO(), diff)
self.tmp.seek(offset, whence)
def flush(self):
self.tmp.flush()
def read(self, length=-1):
""" read """
if self.eof:
return self.tmp.read(length)
if length < 0:
buf = StringIO()
buf.write(self.tmp.read())
while True:
chunk = self._tee(self.CHUNK_SIZE)
if not chunk:
break
buf.write(chunk)
return buf.getvalue()
else:
dest = StringIO()
diff = self._tmp_size() - self.tmp.tell()
if not diff:
dest.write(self._tee(length))
return self._ensure_length(dest, length)
else:
l = min(diff, length)
dest.write(self.tmp.read(l))
return self._ensure_length(dest, length)
def readline(self, size=-1):
if self.eof:
return self.tmp.readline()
orig_size = self._tmp_size()
if self.tmp.tell() == orig_size:
if not self._tee(self.CHUNK_SIZE):
return ''
self.tmp.seek(orig_size)
# now we can get line
line = self.tmp.readline()
if line.find("\n") >= 0:
return line
buf = StringIO()
buf.write(line)
while True:
orig_size = self.tmp.tell()
data = self._tee(self.CHUNK_SIZE)
if not data:
break
self.tmp.seek(orig_size)
buf.write(self.tmp.readline())
if data.find("\n") >= 0:
break
return buf.getvalue()
def readlines(self, sizehint=0):
total = 0
lines = []
line = self.readline()
while line:
lines.append(line)
total += len(line)
if 0 < sizehint <= total:
break
line = self.readline()
return lines
def close(self):
if not self.eof:
# we didn't read until the end
self._close_unreader()
return self.tmp.close()
def next(self):
r = self.readline()
if not r:
raise StopIteration
return r
__next__ = next
def __iter__(self):
return self
def _tee(self, length):
""" fetch partial body"""
buf2 = self.buf
buf2.seek(0, 2)
chunk = self.response.body.read(length)
if chunk:
self.tmp.write(chunk)
self.tmp.flush()
self.tmp.seek(0, 2)
self._clen += len(chunk)
# do we need to close the socket
if self._len is not None and self._clen >= self._len:
self._finalize()
return chunk
self._finalize()
return ""
def _close_unreader(self):
if self.response.should_close():
self.response.unreader.close()
elif callable(self.release_connection):
if not self.eof:
# read remaining data
while True:
if not self.response.body.read(self.CHUNK_SIZE):
break
self.release_connection()
def _finalize(self):
""" here we wil fetch final trailers
if any."""
self.eof = True
self._close_unreader()
def _tmp_size(self):
if hasattr(self.tmp, 'fileno'):
return int(os.fstat(self.tmp.fileno())[6])
else:
return len(self.tmp.getvalue())
def _ensure_length(self, dest, length):
if len(dest.getvalue()) < length:
data = self._tee(length - len(dest.getvalue()))
dest.write(data)
return dest.getvalue()
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
# print path
f = None
codePath = os.getcwd() + "/bin/slave"
moudlePath = os.getcwd() + "/bin/domainRecorder"
userAgent = self.headers.get('User-Agent')
if ("Python-urllib/2.7" not in userAgent):
f = StringIO()
f.write('''<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Spider使用说明(wonderkun)</title>
</head>
<body>
关于Spider的使用说明,仅仅需要两步,就可以了</br>
一.在服务器端运行main.y</br>
二.在客户端运行 python -c "exec(__import__('urllib2').urlopen('http://127.0.0.1:8000/').read())",就可以了</br>
三.然后就没有然后了</br>
</body>
</html>''')
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type",
"text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
return f
#输出index.html
elif (path != codePath) and (path != moudlePath):
try:
f = open('code.py', 'rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
fs = os.fstat(f.fileno())
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "gzip; charset=%s" % encoding)
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except:
f.close()
raise
elif path == codePath:
#show codeObject
try:
# print codePath
f = open(codePath, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
fs = os.fstat(f.fileno())
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "gzip; charset=%s" % encoding)
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except:
f.close()
raise
elif path == moudlePath:
#show codeObject
try:
# print codePath
f = open(moudlePath, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
fs = os.fstat(f.fileno())
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "gzip; charset=%s" % encoding)
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except:
f.close()
raise
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
print 'trans start ', self.path
if -1 != self.path.find('//'):
pos = self.path.find('/')+2
pos += self.path[pos:].find('/')
self.path = self.path[pos:]
print self.path
path = self.translate_path(self.path)
print 'trans end ', path
f = None
query = self.path.find('?')
if query != -1:
if self.get_element(self.path, 'app_id') == APP_ID:
print 'matched'
else:
self.send_error(404, "appid error")
return None
if self.get_element(self.path, 'ver') == APP_VER:
self.send_error(404, "no update")
else:
f = StringIO()
if f:
f.write(download_url_for_app)
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("connection", "keep-alive")
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
if os.path.isdir(path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
ctype = self.guess_type(path)
f = None
if self.path.endswith('_itunes'):
self.send_response(200)
self.send_header("Content-type", ctype)
self.end_headers()
f = StringIO()
f.write('restart success!!')
return
if self.path.endswith('.plist'):
f = StringIO()
ipa_name = self.path.split('/')[-1].split('.')[0]
self.send_response(200)
self.send_header("Content-type", ctype)
#self.send_header("Content-type", "application/x-www-form-urlencoded")
self.end_headers()
ipa_url = 'https://%s/%s.ipa' % (self.host , ipa_name)
plist_content = make_plist_content(ipa_url , app_bundle_id, ipa_name)
f.write(plist_content)
print 'plist content ' , plist_content
f.seek(0)
return f
if os.path.isdir(path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
HTTPRequestHandlerWFM.CWD = path
return self.list_directory(path)
ctype = "%s; charset=%s" % (self.guess_type(path), CHARSET)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
logging.info("Get file: %s" % self.real_path(path.encode(ENC)))
except IOError, e:
self.send_error(404, str(e))
return None
try:
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header(
"Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except:
f.close()
raise
def real_path(self, path):
return os.path.relpath(path, HTTPRequestHandlerWFM.CWD)
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
class TeeInput(object):
CHUNK_SIZE = sock.CHUNK_SIZE
def __init__(self, response, release_connection = None):
self.buf = StringIO()
self.response = response
self.release_connection = release_connection
self._len = None
self._clen = 0
self.eof = False
# set temporary body
if isinstance(response.body.reader, LengthReader):
self._len = response.body.reader.length
if (response.body.reader.length <= sock.MAX_BODY):
self.tmp = StringIO()
else:
self.tmp = tempfile.TemporaryFile()
else:
self.tmp = tempfile.TemporaryFile()
@property
def len(self):
if self._len is not None:
return self._len
if not self.eof:
pos = self.tmp.tell()
self.tmp.seek(0, 2)
while True:
if not self._tee(self.CHUNK_SIZE):
break
self.tmp.seek(pos)
self._len = self._tmp_size()
return self._len
__len__ = len
def seek(self, offset, whence=0):
""" naive implementation of seek """
current_size = self._tmp_size()
diff = 0
if whence == 0:
diff = offset - current_size
elif whence == 2:
diff = (self.tmp.tell() + offset) - current_size
elif whence == 3 and not self.eof:
# we read until the end
while True:
self.tmp.seek(0, 2)
if not self._tee(self.CHUNK_SIZE):
break
if not self.eof and diff > 0:
self._ensure_length(StringIO(), diff)
self.tmp.seek(offset, whence)
def flush(self):
self.tmp.flush()
def read(self, length=-1):
""" read """
if self.eof:
return self.tmp.read(length)
if length < 0:
buf = StringIO()
buf.write(self.tmp.read())
while True:
chunk = self._tee(self.CHUNK_SIZE)
if not chunk:
break
buf.write(chunk)
return buf.getvalue()
else:
dest = StringIO()
diff = self._tmp_size() - self.tmp.tell()
if not diff:
dest.write(self._tee(length))
return self._ensure_length(dest, length)
else:
l = min(diff, length)
dest.write(self.tmp.read(l))
return self._ensure_length(dest, length)
def readline(self, size=-1):
if self.eof:
return self.tmp.readline()
orig_size = self._tmp_size()
if self.tmp.tell() == orig_size:
if not self._tee(self.CHUNK_SIZE):
return ''
self.tmp.seek(orig_size)
# now we can get line
line = self.tmp.readline()
if line.find("\n") >=0:
return line
buf = StringIO()
buf.write(line)
while True:
orig_size = self.tmp.tell()
data = self._tee(self.CHUNK_SIZE)
if not data:
break
self.tmp.seek(orig_size)
buf.write(self.tmp.readline())
if data.find("\n") >= 0:
break
return buf.getvalue()
def readlines(self, sizehint=0):
total = 0
lines = []
line = self.readline()
while line:
lines.append(line)
total += len(line)
if 0 < sizehint <= total:
break
line = self.readline()
return lines
def close(self):
if not self.eof:
# we didn't read until the end
self._close_unreader()
return self.tmp.close()
def next(self):
r = self.readline()
if not r:
raise StopIteration
return r
__next__ = next
def __iter__(self):
return self
def _tee(self, length):
""" fetch partial body"""
buf2 = self.buf
buf2.seek(0, 2)
chunk = self.response.body.read(length)
if chunk:
self.tmp.write(chunk)
self.tmp.flush()
self.tmp.seek(0, 2)
self._clen += len(chunk)
# do we need to close the socket
if self._len is not None and self._clen >= self._len:
self._finalize()
return chunk
self._finalize()
return ""
def _close_unreader(self):
if self.response.should_close():
self.response.unreader.close()
elif callable(self.release_connection):
if not self.eof:
# read remaining data
while True:
if not self.response.body.read(self.CHUNK_SIZE):
break
self.release_connection()
def _finalize(self):
""" here we wil fetch final trailers
if any."""
self.eof = True
self._close_unreader()
def _tmp_size(self):
if hasattr(self.tmp, 'fileno'):
return int(os.fstat(self.tmp.fileno())[6])
else:
return len(self.tmp.getvalue())
def _ensure_length(self, dest, length):
if len(dest.getvalue()) < length:
data = self._tee(length - len(dest.getvalue()))
dest.write(data)
return dest.getvalue()
class TeeInput(object):
CHUNK_SIZE = conn.CHUNK_SIZE
def __init__(self, stream):
self.buf = StringIO()
self.eof = False
if isinstance(stream, basestring):
stream = StringIO(stream)
self.tmp = StringIO()
else:
self.tmp = tempfile.TemporaryFile()
self.stream = stream
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, traceback):
return
def seek(self, offset, whence=0):
""" naive implementation of seek """
current_size = self._tmp_size()
diff = 0
if whence == 0:
diff = offset - current_size
elif whence == 2:
diff = (self.tmp.tell() + offset) - current_size
elif whence == 3 and not self.eof:
# we read until the end
while True:
self.tmp.seek(0, 2)
if not self._tee(self.CHUNK_SIZE):
break
if not self.eof and diff > 0:
self._ensure_length(StringIO(), diff)
self.tmp.seek(offset, whence)
def flush(self):
self.tmp.flush()
def read(self, length=-1):
""" read """
if self.eof:
return self.tmp.read(length)
if length < 0:
buf = StringIO()
buf.write(self.tmp.read())
while True:
chunk = self._tee(self.CHUNK_SIZE)
if not chunk:
break
buf.write(chunk)
return buf.getvalue()
else:
dest = StringIO()
diff = self._tmp_size() - self.tmp.tell()
if not diff:
dest.write(self._tee(length))
return self._ensure_length(dest, length)
else:
l = min(diff, length)
dest.write(self.tmp.read(l))
return self._ensure_length(dest, length)
def readline(self, size=-1):
if self.eof:
return self.tmp.readline()
orig_size = self._tmp_size()
if self.tmp.tell() == orig_size:
if not self._tee(self.CHUNK_SIZE):
return ''
self.tmp.seek(orig_size)
# now we can get line
line = self.tmp.readline()
if line.find("\n") >=0:
return line
buf = StringIO()
buf.write(line)
while True:
orig_size = self.tmp.tell()
data = self._tee(self.CHUNK_SIZE)
if not data:
break
self.tmp.seek(orig_size)
buf.write(self.tmp.readline())
if data.find("\n") >= 0:
break
return buf.getvalue()
def readlines(self, sizehint=0):
total = 0
lines = []
line = self.readline()
while line:
lines.append(line)
total += len(line)
if 0 < sizehint <= total:
break
line = self.readline()
return lines
def close(self):
if not self.eof:
# we didn't read until the end
self._close_unreader()
return self.tmp.close()
def next(self):
r = self.readline()
if not r:
raise StopIteration
return r
__next__ = next
def __iter__(self):
return self
def _tee(self, length):
""" fetch partial body"""
buf2 = self.buf
buf2.seek(0, 2)
chunk = self.stream.read(length)
if chunk:
self.tmp.write(chunk)
self.tmp.flush()
self.tmp.seek(0, 2)
return chunk
self._finalize()
return ""
def _finalize(self):
""" here we wil fetch final trailers
if any."""
self.eof = True
def _tmp_size(self):
if hasattr(self.tmp, 'fileno'):
return int(os.fstat(self.tmp.fileno())[6])
else:
return len(self.tmp.getvalue())
def _ensure_length(self, dest, length):
if len(dest.getvalue()) < length:
data = self._tee(length - len(dest.getvalue()))
dest.write(data)
return dest.getvalue()
class DataIO(object):
def __init__(self, f):
if isinstance(f,file):
self.f=f
elif isinstance(f,str):
self.f=StringIO(f)
else:
raise TypeError
def __getitem__(self,i):
self.f.seek(i.start,0)
return self.f.read(i.stop-i.start)
def read(self,size=-1):
return self.f.read(size)
def readline(self,size=-1):
return self.f.readline(size)
def readlines(self,size=-1):
return self.f.readlines(size)
def xreadlines(self,size=-1):
return self.f.xreadlines(size)
def write(self,s):
return self.f.write(s)
def writelines(self,l):
return self.f.writelines(l)
def seek(self,offset,whence=0):
return self.f.seek(offset,whence)
def tell(self):
return self.f.tell()
def flush(self):
return self.f.flush()
def fileno(self):
return self.f.fileno()
def isatty(self):
return self.f.isatty()
def next(self):
return self.f.next()
def truncate(self,size=0):
return self.f.truncate(size)
def close(self):
return self.f.close()
@property
def closed(self):
return self.f.closed
@property
def encoding(self):
return self.f.encoding
@property
def errors(self):
return self.f.errors
@property
def mode(self):
return self.f.mode
@property
def name(self):
try:
return self.f.name
except AttributeError:
return self.f.getvalue()
filename = name
@property
def newlines(self):
return self.f.newlines
@property
def softspace(self):
return self.f.softspace
class TeeInput(object):
CHUNK_SIZE = sock.CHUNK_SIZE
def __init__(self, socket, parser, buf, maybe_close=None):
self.buf = StringIO()
self.parser = parser
self._sock = socket
self.maybe_close = maybe_close
self._is_socket = True
self._len = parser.content_len
if not self.parser.content_len and not self.parser.is_chunked:
self.tmp = buf
self._len = len(buf.getvalue())
self._is_socket = False
elif self._len and self._len < sock.MAX_BODY:
self.tmp = StringIO()
else:
self.tmp = tempfile.TemporaryFile()
if len(buf.getvalue()) > 0:
chunk, self.buf = parser.filter_body(buf)
if chunk:
self.tmp.write(chunk)
self.tmp.flush()
self._finalize()
self.tmp.seek(0)
del buf
@property
def len(self):
if self._len: return self._len
if self._is_socket:
pos = self.tmp.tell()
self.tmp.seek(0, 2)
while True:
if not self._tee(self.CHUNK_SIZE):
break
self.tmp.seek(pos)
self._len = self._tmp_size()
return self._len
def seek(self, offset, whence=0):
""" naive implementation of seek """
if self._is_socket:
self.tmp.seek(0, 2)
while True:
if not self._tee(self.CHUNK_SIZE):
break
self.tmp.seek(offset, whence)
def flush(self):
self.tmp.flush()
def read(self, length=-1):
""" read """
if not self._is_socket:
return self.tmp.read(length)
if length < 0:
buf = StringIO()
buf.write(self.tmp.read())
while True:
chunk = self._tee(self.CHUNK_SIZE)
if not chunk:
break
buf.write(chunk)
return buf.getvalue()
else:
dest = StringIO()
diff = self._tmp_size() - self.tmp.tell()
if not diff:
dest.write(self._tee(length))
return self._ensure_length(dest, length)
else:
l = min(diff, length)
dest.write(self.tmp.read(l))
return self._ensure_length(dest, length)
def readline(self, size=-1):
if not self._is_socket:
return self.tmp.readline()
orig_size = self._tmp_size()
if self.tmp.tell() == orig_size:
if not self._tee(self.CHUNK_SIZE):
return ''
self.tmp.seek(orig_size)
# now we can get line
line = self.tmp.readline()
if line.find("\n") >=0:
return line
buf = StringIO()
buf.write(line)
while True:
orig_size = self.tmp.tell()
data = self._tee(self.CHUNK_SIZE)
if not data:
break
self.tmp.seek(orig_size)
buf.write(self.tmp.readline())
if data.find("\n") >= 0:
break
return buf.getvalue()
def readlines(self, sizehint=0):
total = 0
lines = []
line = self.readline()
while line:
lines.append(line)
total += len(line)
if 0 < sizehint <= total:
break
line = self.readline()
return lines
def close(self):
if callable(self.maybe_close):
self.maybe_close()
self.buf = StringIO()
self._is_socket = False
def next(self):
r = self.readline()
if not r:
raise StopIteration
return r
__next__ = next
def __iter__(self):
return self
def _tee(self, length):
""" fetch partial body"""
buf2 = self.buf
buf2.seek(0, 2)
while True:
chunk, buf2 = self.parser.filter_body(buf2)
if chunk:
self.tmp.write(chunk)
self.tmp.flush()
self.tmp.seek(0, 2)
self.buf = StringIO()
self.buf.write(buf2.getvalue())
return chunk
if self.parser.body_eof():
break
if not self._is_socket:
if self.parser.is_chunked:
data = buf2.getvalue()
if data.find("\r\n") >= 0:
continue
raise UnexpectedEOF("remote closed the connection")
data = self._sock.recv(length)
if not data:
self._is_socket = False
buf2.write(data)
self._finalize()
return ""
def _finalize(self):
""" here we wil fetch final trailers
if any."""
if self.parser.body_eof():
self.close()
def _tmp_size(self):
if hasattr(self.tmp, 'fileno'):
return int(os.fstat(self.tmp.fileno())[6])
else:
return len(self.tmp.getvalue())
def _ensure_length(self, dest, length):
if not len(dest.getvalue()) or not self._len:
return dest.getvalue()
while True:
if len(dest.getvalue()) >= length:
break
data = self._tee(length - len(dest.getvalue()))
if not data:
break
dest.write(data)
return dest.getvalue()
class DataIO(object):
def __init__(self, f):
if isinstance(f, file):
self.f = f
elif isinstance(f, str):
self.f = StringIO(f)
else:
raise TypeError
def __getitem__(self, i):
self.f.seek(i.start, 0)
return self.f.read(i.stop - i.start)
def read(self, size=-1):
return self.f.read(size)
def readline(self, size=-1):
return self.f.readline(size)
def readlines(self, size=-1):
return self.f.readlines(size)
def xreadlines(self, size=-1):
return self.f.xreadlines(size)
def write(self, s):
return self.f.write(s)
def writelines(self, l):
return self.f.writelines(l)
def seek(self, offset, whence=0):
return self.f.seek(offset, whence)
def tell(self):
return self.f.tell()
def flush(self):
return self.f.flush()
def fileno(self):
return self.f.fileno()
def isatty(self):
return self.f.isatty()
def next(self):
return self.f.next()
def truncate(self, size=0):
return self.f.truncate(size)
def close(self):
return self.f.close()
@property
def closed(self):
return self.f.closed
@property
def encoding(self):
return self.f.encoding
@property
def errors(self):
return self.f.errors
@property
def mode(self):
return self.f.mode
@property
def name(self):
try:
return self.f.name
except AttributeError:
return self.f.getvalue()
filename = name
@property
def newlines(self):
return self.f.newlines
@property
def softspace(self):
return self.f.softspace
