def read_message(f):
"""Read a packet
return msg received, if error happens, raise socket.error
"""
# Read two bytes: message version and length
data = f.recv(8)
if not data:
raise socket.error('socket recv error')
ver, length = unpack('!ii', data)
# Read message body
data = f.recv(length)
if not data:
raise socket.error('socket recv error')
msg = OODict(cPickle.loads(data))
if 'payload_length' not in msg: # Simple message
return msg
# Read payload
done_len = 0
payload = ''
while True:
data = f.recv(BUFFER_SIZE)
if not data:
raise socket.error('socket recv error')
done_len += len(data)
payload += data
if done_len >= msg.payload_length:
break
debug('protocol.read_message: payload-length %d get %d', msg.payload_length, len(payload))
msg.payload = payload
del msg['payload_length']
return msg
def _parse_node(self, node):
tree = OODict()
# Save value
value = node.text
if isinstance(value, str):
value = value.strip() # Only strip strings
tree.value = value
# Save attributes
attrs = {}
for k,v in node.attrib.items():
attrs.update(self._make_dict(k, v))
if attrs:
tree['attrs'] = attrs
#Save childrens
for child in node.getchildren():
ctag = child.tag
ctree = self._parse_node(child)
cdict = self._make_dict(ctag, ctree)
if ctag not in tree: # First time found
tree.update(cdict)
continue
old = tree[ctag]
if not isinstance(old, list):
tree[ctag] = [old] # Multi entries, change to list
tree[ctag].append(ctree) # Add new entry
return tree
def load_chunk(self, chunk, dev):
"""Read chunk from device, return a chunk object
@fid
@cid
@version
@dev
"""
file = self._get_chunk_path(chunk, dev)
if not os.path.isfile(file):
raise IOError('chunk lost or stale')
fp = open(file, 'r')
try:
header = OODict(eval(fp.read(CHUNK_HEADER_SIZE).strip('\x00')))
except:
raise IOError('chunk header corrupt')
if not header:
raise IOError('chunk header corrupt')
data = fp.read(header.size)
if len(data) != header.size:
raise IOError('chunk data lost')
if header.algo == 'adler32' and zlib.adler32(data) != header.checksum:
raise IOError('chunk data corrupt')
if header.algo == 'sha1' and hashlib.sha1(data).hexdigest() != header.checksum:
raise IOError('chunk data corrupt')
fp.close()
header.data = data
return header
def get_album_photos(self, album):
"""Get all photos of an album
Return: dict(url, title, photos)
"""
data = OODict()
data.url = self._get_url('photos/album', album)
data.title = self._get_title(data.url)
data.photos = self._extract_multi_pages(self.PATTERNS['photo'], data.url)
return data
def _make_dict(self, tag, value):
"""Generate a new dict with tag and value
If tag is like '{http://cs.sfsu.edu/csc867/myscheduler}patients',
split it first to: http://cs.sfsu.edu/csc867/myscheduler, patients
"""
tmp = value
result = re.compile("\{(.*)\}(.*)").search(tag)
if result:
tmp = OODict()
tmp.xmlns, tag = result.groups() # We have a namespace!
tmp.value = value
return OODict({tag: tmp})
def __init__(self, data):
f = StreamUnpacker(data)
self.z_acl_extern_obj = f.uint64()
self.z_acl_count = f.uint32()
self.z_acl_version = f.uint16()
self.z_ace_data = []
for data in split_records(data[16:], ACL_T_SIZE):
acl = OODict()
f = StreamUnpacker(data)
acl.a_who = f.uint64()
acl.a_access_mask = f.uint32()
acl.a_flags, acl.a_type = f.repeat('uint16', 2)
self.z_ace_data.append(acl)
def _enqueue(self, fp, data, length, callback, queue, mask):
fn = fp.fileno()
io = OODict()
io.fn = fn
io.ctime = time.time()
io.data = data
io.length = length
io.callback = callback
io.done_cv = threading.Condition()
io.done = False
if fn not in self.io_queue:
self.io_queue[fn] = OODict()
self.io_queue[fn].fp = fp
self.io_queue[fn].mask = 0
self.io_queue[fn].errs = 0
fp.setblocking(0)
self.epoll.register(fn, self.io_queue[fn].mask)
if queue not in self.io_queue[fn]:
self.io_queue[fn][queue] = []
if self.io_queue[fn][queue]:
self.io_queue[fn][queue].append(io)
else:
self.io_queue[fn][queue] = [io]
self.io_queue[fn].mask |= mask
self.epoll.modify(fn, self.io_queue[fn].mask)
print queue, io
return io
def __init__(self, data):
f = StreamUnpacker(data)
self.z_acl_extern_obj = f.uint64()
self.z_acl_count = f.uint32()
self.z_acl_version = f.uint16()
self.z_ace_data = []
for data in split_records(data[16:], ACL_T_SIZE):
acl = OODict()
f = StreamUnpacker(data)
acl.a_who = f.uint64()
acl.a_access_mask = f.uint32()
acl.a_flags, acl.a_type = f.repeat('uint16', 2)
self.z_ace_data.append(acl)
def processing(self, req):
response = OODict()
response._id = req._id
try:
result = self.handle_req(req) # Must set req._status
if result is None:
return None # Request not done, need to queue back
# For complicated calls, read for example, should return a tuple: value, payload
if isinstance(result, tuple):
response.value, response.payload = result
else:
response.value = result
except RequestHandleError, err:
response.error = str(err)
def unpack_message_body(data):
"""Unpack message body from data, set msg._end if we carry payload"""
l = len(data)
if l < 8:
return None # version and length not complete
ver, length = unpack('!ii', data[:8])
msg_end = 8 + length
# Read message body
if l < msg_end:
return None # Message body not complete
msg = OODict(cPickle.loads(data[8: msg_end]))
if 'payload_length' in msg: # For payload
msg._end = msg_end
return msg
def _parse_dva(self, data):
dva = OODict()
su = StreamUnpacker(data)
i = su.uint64()
dva.asize = get_bits(i, 0, 24) << SPA_MINBLOCKSHIFT
dva.grid = get_bits(i, 24, 8)
dva.vdev = get_bits(i, 32, 32)
i = su.uint64()
dva.offset = get_bits(i, 0, 63) << SPA_MINBLOCKSHIFT
if get_bits(i, 63, 1):
dva.G = True
else:
dva.G = False
return dva
def call(self, dest, method, **args):
# How to detect socket errors? how to resend message and reconnect
# socket?
# Connect socket
if dest not in self._locks:
self._locks[dest] = thread.allocate_lock()
self._locks[dest].acquire()
if dest not in self._sockets:
self._connect(dest)
# Pack message
req = OODict()
req.method = method
for k, v in args.items():
req[k] = v
# Lock id here
self._id_lock.acquire()
req._id = self._id
self._id += 1
self._id_lock.release()
retry = 0
while True:
try:
protocol.send_message(self._sockets[dest], req)
break
except socket.error, err:
# There must be something wrong with this socket
# Create a new one
self._connect(dest)
# Only retry when connected,
if retry >= self._send_error_retry:
self._locks[dest].release()
raise exception.NoMoreRetryError('socket send error')
# Do not need sleep here
retry += 1
def _parse_dva(self, data):
dva = OODict()
su = StreamUnpacker(data)
i = su.uint64()
dva.asize = get_bits(i, 0, 24) << SPA_MINBLOCKSHIFT
dva.grid = get_bits(i, 24, 8)
dva.vdev = get_bits(i, 32, 32)
i = su.uint64()
dva.offset = get_bits(i, 0, 63) << SPA_MINBLOCKSHIFT
if get_bits(i, 63, 1):
dva.G = True
else:
dva.G = False
return dva
def strip(cls, nvlist):
"""
Strip a nvlist dict to an simple OODict
"""
striped = OODict()
for pair in nvlist['nvpairs']:
pt = pair['type']
pn = pair['name']
pv = pair['value']
#print pt, pn, pv
if 'NVLIST_ARRAY' in pt:
striped[pn] = []
for v in pv:
striped[pn].append(cls.strip(v))
else:
if 'NVLIST' in pt:
striped[pn] = cls.strip(pv)
else:
striped[pn] = pv
return striped
def accept_clients(self):
"""Exception will be raised if there are no more connections to
accept:
[Errno 11] Resource temporarily unavailable
"""
while True:
sk, addr = self.socket.accept()
if not sk or not addr:
break # No more connections
sk.setblocking(0) # Non blocking client socket
self.epoll.register(sk.fileno(), select.EPOLLIN) # Poll in
# Init conn here
client = OODict()
client.addr = addr
client.sk = sk
client.request = None
client.request_data = ''
client.wrong_epollin_events = 0
client.response = None
self.clients[sk.fileno()] = client # Using fn as key
debug('%s connected', addr)
