def _read_metadata(self):
"""Read the transaction log from the cache directory.
Reads the pickled log entries and re-creates the cache's
current contents and use_order from the log.
A version number is included, but currently we only assert
that a the version number read is the same as the current
version number.
"""
logpath = os.path.join(self.directory, 'LOG')
try:
log = open(logpath)
except IOError:
# now what happens if there is an error here?
log = open(logpath, 'w')
log.close()
return
for line in log.readlines():
try:
kind = line[0:1]
if kind == '2': # use update
key = line[2:-1]
self.use_order.remove(key)
self.use_order.append(key)
elif kind == '1': # delete
key = line[2:-1]
if self.items.has_key(key):
self.size = self.size - self.items[key].size
del self.items[key]
del self.manager.items[key]
self.use_order.remove(key)
Assert(not key in self.use_order)
elif kind == '0': # add
newentry = DiskCacheEntry(self)
newentry.parse(line[2:-1])
if not self.items.has_key(newentry.key):
self.use_order.append(newentry.key)
newentry.cache = self
self.items[newentry.key] = newentry
self.manager.items[newentry.key] = newentry
self.size = self.size + newentry.size
elif kind == '3': # version (hopefully first)
ver = line[2:-1]
if ver not in self.log_ok_versions:
### clear out anything we might have read
### and bail. this is an old log file.
if len(self.use_order) > 0:
self.use_order = []
for key in self.items.keys():
del self.items[key]
del self.manager.items[key]
self.size = 0
return
Assert(ver in self.log_ok_versions)
except IndexError:
# ignore this line
pass
def getdata(self, maxbytes):
Assert(self.stage == DATA)
data = self.item.getdata(self.offset, maxbytes)
self.offset = self.offset + len(data)
if not data:
self.close()
return data
def pollmeta(self, timeout=0):
Assert(self.state == META)
sock = self.h.sock
try:
if not select.select([sock], [], [], timeout)[0]:
return "waiting for server response", 0
except select.error as msg:
raise IOError, msg, sys.exc_traceback
try:
new = sock.recv(1024)
except socket.error as msg:
raise IOError, msg, sys.exc_traceback
if not new:
return "EOF in server response", 1
self.readahead = self.readahead + new
if '\n' not in new:
return "receiving server response", 0
if not self.line1seen:
i = str.find(self.readahead, '\n')
if i < 0:
return "receiving server response", 0
self.line1seen = 1
line = self.readahead[:i + 1]
if replyprog.match(line) < 0:
return "received non-HTTP/1.0 server response", 1
i = endofheaders.search(self.readahead)
if i >= 0:
return "received server response", 1
return "receiving server response", 0
def open(self):
Assert(self.state == WAIT)
resturl, method, params, data = self.args
if data:
Assert(method == "POST")
else:
Assert(method in ("GET", "POST"))
if type(resturl) == type(()):
host, selector = resturl # For proxy interface
else:
host, selector = splithost(resturl)
if not host:
raise IOError("no host specified in URL")
i = str.find(host, '@')
if i >= 0:
user_passwd, host = host[:i], host[i + 1:]
else:
user_passwd = None
if user_passwd:
import base64
auth = str.strip(base64.encodestring(user_passwd))
else:
auth = None
self.h = MyHTTP(host)
self.h.putrequest(method, selector)
self.h.putheader('User-agent', GRAILVERSION)
if auth:
self.h.putheader('Authorization', 'Basic %s' % auth)
if not params.has_key('host'):
self.h.putheader('Host', host)
if not params.has_key('accept-encoding'):
encodings = Reader.get_content_encodings()
if encodings:
encodings.sort()
self.h.putheader('Accept-Encoding', str.join(encodings, ", "))
for key, value in params.items():
if key[:1] != '.':
self.h.putheader(key, value)
self.h.putheader('Accept', '*/*')
self.h.endheaders()
if data:
self.h.send(data)
self.readahead = ""
self.state = META
self.line1seen = 0
if self.reader_callback:
self.reader_callback()
def getdata(self, maxbytes):
Assert(self.state == nullAPI.DATA)
split_pos = min(maxbytes, len(self.__data))
data = self.__data[:split_pos]
self.__data = self.__data[split_pos:]
if not data:
self.state = nullAPI.DONE
return data
def getmeta(self):
Assert(self.state == nullAPI.META)
self.state = nullAPI.DATA
headers = {"content-type": self.__ctype,
"content-length": repr(len(self.__data)),
}
if self.__data:
return 200, "Ready", headers
return 204, "No content", headers
def decref(self):
Assert(self.refcnt > 0)
self.refcnt = self.refcnt - 1
self.cache_update()
if self.refcnt == 0:
if self.stage == DONE:
self.finish()
else:
self.abort()
def getmeta(self):
Assert(self.state == META)
if not self.readahead:
x, y = self.pollmeta(None)
while not y:
x, y = self.pollmeta(None)
file = io.io(self.readahead)
errcode, errmsg, headers = self.h.getreply(file)
self.state = DATA
self.readahead = file.read()
return errcode, errmsg, headers
def getdata(self, offset, maxbytes):
Assert(offset >= 0)
Assert(maxbytes > 0)
while self.stage == DATA and offset >= self.datalen:
buf = self.api.getdata(maxbytes)
if not buf:
self.finish()
self.complete = 1
else:
l = len(buf)
if l > maxbytes:
# we got more than we wanted,
# so split into two strings (avoid search next time)
self.data.append(buf[:maxbytes])
self.datamap[offset] = len(self.data) - 1
self.data.append(buf[maxbytes:])
self.datamap[offset+maxbytes] = len(self.data) - 1
else:
self.data.append(buf)
self.datamap[offset] = len(self.data) - 1
self.datalen = self.datalen + l
try:
# the common case
chunk = self.data[self.datamap[offset]]
if len(chunk) > maxbytes:
return chunk[0:maxbytes]
else:
return chunk
except KeyError:
if self.stage == META:
self.meta = self.api.getmeta()
self.stage = DATA
elif self.complete == 1 and offset >= self.datalen:
return ''
chunk_key, delta = self._getdata_search_string_list(offset)
chunk = self.data[self.datamap[chunk_key]]
return chunk[delta:]
def getdata(self, maxbytes):
Assert(self.state == DATA)
if self.readahead:
data = self.readahead[:maxbytes]
self.readahead = self.readahead[maxbytes:]
return data
try:
data = self.h.sock.recv(maxbytes)
except socket.error as msg:
raise IOError, msg, sys.exc_traceback
if not data:
self.state = DONE
# self.close()
return data
def getdata(self, maxbytes):
if self.state == EOF:
self.state = DONE
return ""
Assert(self.state == DATA)
data = self.sock.recv(maxbytes)
if self.debuglevel > 4: print("*data*", repr(data))
if not data:
self.state = DONE
if self.isdir:
self.addlistingdata(data)
data = self.getlistingdata()
if self.state == DONE and data:
self.state = EOF # Allow one more call
return data
def getmeta(self):
Assert(self.state == META)
self.state = DATA
headers = {}
if self.isdir:
if self.url and self.url[-1:] != '/':
self.url = self.url + '/'
self.content_type = 'text/html'
self.content_encoding = None
if self.content_type:
headers['content-type'] = self.content_type
if self.content_encoding:
headers['content-encoding'] = self.content_encoding
if self.content_length:
headers['content-length'] = repr(self.content_length)
self.lines = [] # Only used of self.isdir
return 200, "OK", headers
def getdata(self, maxbytes):
Assert(self.state == DATA)
data = self.fp.read(maxbytes)
if not data:
self.state = DONE
return data
def expire(self,key):
"""Should not be used."""
Assert('night' == 'day')
Assert(self.items.has_key(key))
self.items[key].evict()
def get(self,key):
"""Update and log use_order."""
Assert(self.items.has_key(key))
self.use_order.remove(key)
self.use_order.append(key)
self.log_use_order(key)
def pollmeta(self):
Assert(self.state == META)
return "Ready", 1
def getdata(self, maxbytes):
Assert(self.state == DATA)
self.state = DONE
return ""
def polldata(self):
Assert(self.state == DATA)
return "Ready", 1
def polldata(self):
Assert(self.state in (nullAPI.META, nullAPI.DATA))
return "Ready", 1
def polldata(self):
Assert(self.stage == DATA)
return self.item.polldata()
def getmeta(self):
Assert(self.stage == META)
meta = self.item.getmeta()
self.stage = DATA
return meta
def pollmeta(self):
Assert(self.stage == META)
return self.item.pollmeta()
def getmeta(self):
Assert(self.state == META)
self.state = DATA
if self.redirect:
return 301, "Redirect to absolute pathname", {"location": self.url}
return 200, "OK", self.headers
def __init__(self, url, method, params):
Assert(method == 'GET')
netloc, path = splithost(url)
if not netloc: raise IOError('ftp error', 'no host given')
host, port = splitport(netloc)
user, host = splituser(host)
if user: user, passwd = splitpasswd(user)
else: passwd = None
host = socket.gethostbyname(host)
if port:
try:
port = int(port)
except ValueError:
raise IOError('ftp error', 'bad port')
else:
port = ftplib.FTP_PORT
path, attrs = splitattr(path)
self.url = "ftp://%s%s" % (netloc, path)
dirs = str.split(path, '/')
dirs, file = dirs[:-1], dirs[-1]
self.content_length = None
if not file:
self.content_type, self.content_encoding = None, None
type = 'd'
else:
self.content_type, self.content_encoding = app.guess_type(file)
if self.content_encoding:
type = 'i'
elif self.content_type and self.content_type[:5] == 'text/':
type = 'a'
elif file[-1] == '/':
type = 'd'
else:
type = 'i'
if dirs and not dirs[0]: dirs = dirs[1:]
key = (user, host, port, str.join(dirs, '/'))
self.debuglevel = None
try:
if not ftpcache.has_key(key):
ftpcache[key] = []
for attr in attrs:
[attr, value] = map(str.lower, splitvalue(attr))
if attr == 'type' and value in ('a', 'i', 'd'):
type = value
elif attr == 'debug':
try:
self.debuglevel = int(value)
except ValueError:
pass
candidates = ftpcache[key]
for cand in candidates:
if not cand.busy():
break
else:
cand = ftpwrapper(user, passwd, host, port, dirs,
self.debuglevel)
candidates.append(cand)
# XXX Ought to clean the cache every once in a while
self.cand = cand
self.sock, self.isdir = cand.retrfile(file, type)
self.content_length = cand.content_length
except ftplib.all_errors as msg:
raise IOError('ftp error', msg)
self.state = META
def polldata(self):
Assert(self.state == DATA)
if self.readahead:
return "processing readahead data", 1
return ("waiting for data", len(select.select([self], [], [], 0)[0]))
def getmeta(self):
Assert(self.state == META)
self.state = DATA
return 204, "No content", {}
def test_setup(self):
Assert(self.browser).is_class('API_Browser')
Assert(self.vis_js.web_root).contains('serverless-render/src/web_root')
Assert(self.browser.sync__url()).is_equal('about:blank')
Dev.print(self.browser.sync__url())
def polldata(self):
Assert(self.state in (EOF, DATA))
return "Ready", 1