def read_message(f): """Read a packet return msg received, if error happens, raise socket.error """ # Read two bytes: message version and length data = f.recv(8) if not data: raise socket.error('socket recv error') ver, length = unpack('!ii', data) # Read message body data = f.recv(length) if not data: raise socket.error('socket recv error') msg = OODict(cPickle.loads(data)) if 'payload_length' not in msg: # Simple message return msg # Read payload done_len = 0 payload = '' while True: data = f.recv(BUFFER_SIZE) if not data: raise socket.error('socket recv error') done_len += len(data) payload += data if done_len >= msg.payload_length: break debug('protocol.read_message: payload-length %d get %d', msg.payload_length, len(payload)) msg.payload = payload del msg['payload_length'] return msg
def _parse_node(self, node): tree = OODict() # Save value value = node.text if isinstance(value, str): value = value.strip() # Only strip strings tree.value = value # Save attributes attrs = {} for k,v in node.attrib.items(): attrs.update(self._make_dict(k, v)) if attrs: tree['attrs'] = attrs #Save childrens for child in node.getchildren(): ctag = child.tag ctree = self._parse_node(child) cdict = self._make_dict(ctag, ctree) if ctag not in tree: # First time found tree.update(cdict) continue old = tree[ctag] if not isinstance(old, list): tree[ctag] = [old] # Multi entries, change to list tree[ctag].append(ctree) # Add new entry return tree
def load_chunk(self, chunk, dev): """Read chunk from device, return a chunk object @fid @cid @version @dev """ file = self._get_chunk_path(chunk, dev) if not os.path.isfile(file): raise IOError('chunk lost or stale') fp = open(file, 'r') try: header = OODict(eval(fp.read(CHUNK_HEADER_SIZE).strip('\x00'))) except: raise IOError('chunk header corrupt') if not header: raise IOError('chunk header corrupt') data = fp.read(header.size) if len(data) != header.size: raise IOError('chunk data lost') if header.algo == 'adler32' and zlib.adler32(data) != header.checksum: raise IOError('chunk data corrupt') if header.algo == 'sha1' and hashlib.sha1(data).hexdigest() != header.checksum: raise IOError('chunk data corrupt') fp.close() header.data = data return header
def get_album_photos(self, album): """Get all photos of an album Return: dict(url, title, photos) """ data = OODict() data.url = self._get_url('photos/album', album) data.title = self._get_title(data.url) data.photos = self._extract_multi_pages(self.PATTERNS['photo'], data.url) return data
def _make_dict(self, tag, value): """Generate a new dict with tag and value If tag is like '{http://cs.sfsu.edu/csc867/myscheduler}patients', split it first to: http://cs.sfsu.edu/csc867/myscheduler, patients """ tmp = value result = re.compile("\{(.*)\}(.*)").search(tag) if result: tmp = OODict() tmp.xmlns, tag = result.groups() # We have a namespace! tmp.value = value return OODict({tag: tmp})
def __init__(self, data): f = StreamUnpacker(data) self.z_acl_extern_obj = f.uint64() self.z_acl_count = f.uint32() self.z_acl_version = f.uint16() self.z_ace_data = [] for data in split_records(data[16:], ACL_T_SIZE): acl = OODict() f = StreamUnpacker(data) acl.a_who = f.uint64() acl.a_access_mask = f.uint32() acl.a_flags, acl.a_type = f.repeat('uint16', 2) self.z_ace_data.append(acl)
def _enqueue(self, fp, data, length, callback, queue, mask): fn = fp.fileno() io = OODict() io.fn = fn io.ctime = time.time() io.data = data io.length = length io.callback = callback io.done_cv = threading.Condition() io.done = False if fn not in self.io_queue: self.io_queue[fn] = OODict() self.io_queue[fn].fp = fp self.io_queue[fn].mask = 0 self.io_queue[fn].errs = 0 fp.setblocking(0) self.epoll.register(fn, self.io_queue[fn].mask) if queue not in self.io_queue[fn]: self.io_queue[fn][queue] = [] if self.io_queue[fn][queue]: self.io_queue[fn][queue].append(io) else: self.io_queue[fn][queue] = [io] self.io_queue[fn].mask |= mask self.epoll.modify(fn, self.io_queue[fn].mask) print queue, io return io
def processing(self, req): response = OODict() response._id = req._id try: result = self.handle_req(req) # Must set req._status if result is None: return None # Request not done, need to queue back # For complicated calls, read for example, should return a tuple: value, payload if isinstance(result, tuple): response.value, response.payload = result else: response.value = result except RequestHandleError, err: response.error = str(err)
def unpack_message_body(data): """Unpack message body from data, set msg._end if we carry payload""" l = len(data) if l < 8: return None # version and length not complete ver, length = unpack('!ii', data[:8]) msg_end = 8 + length # Read message body if l < msg_end: return None # Message body not complete msg = OODict(cPickle.loads(data[8: msg_end])) if 'payload_length' in msg: # For payload msg._end = msg_end return msg
def _parse_dva(self, data): dva = OODict() su = StreamUnpacker(data) i = su.uint64() dva.asize = get_bits(i, 0, 24) << SPA_MINBLOCKSHIFT dva.grid = get_bits(i, 24, 8) dva.vdev = get_bits(i, 32, 32) i = su.uint64() dva.offset = get_bits(i, 0, 63) << SPA_MINBLOCKSHIFT if get_bits(i, 63, 1): dva.G = True else: dva.G = False return dva
def call(self, dest, method, **args): # How to detect socket errors? how to resend message and reconnect # socket? # Connect socket if dest not in self._locks: self._locks[dest] = thread.allocate_lock() self._locks[dest].acquire() if dest not in self._sockets: self._connect(dest) # Pack message req = OODict() req.method = method for k, v in args.items(): req[k] = v # Lock id here self._id_lock.acquire() req._id = self._id self._id += 1 self._id_lock.release() retry = 0 while True: try: protocol.send_message(self._sockets[dest], req) break except socket.error, err: # There must be something wrong with this socket # Create a new one self._connect(dest) # Only retry when connected, if retry >= self._send_error_retry: self._locks[dest].release() raise exception.NoMoreRetryError('socket send error') # Do not need sleep here retry += 1
def strip(cls, nvlist): """ Strip a nvlist dict to an simple OODict """ striped = OODict() for pair in nvlist['nvpairs']: pt = pair['type'] pn = pair['name'] pv = pair['value'] #print pt, pn, pv if 'NVLIST_ARRAY' in pt: striped[pn] = [] for v in pv: striped[pn].append(cls.strip(v)) else: if 'NVLIST' in pt: striped[pn] = cls.strip(pv) else: striped[pn] = pv return striped
def accept_clients(self): """Exception will be raised if there are no more connections to accept: [Errno 11] Resource temporarily unavailable """ while True: sk, addr = self.socket.accept() if not sk or not addr: break # No more connections sk.setblocking(0) # Non blocking client socket self.epoll.register(sk.fileno(), select.EPOLLIN) # Poll in # Init conn here client = OODict() client.addr = addr client.sk = sk client.request = None client.request_data = '' client.wrong_epollin_events = 0 client.response = None self.clients[sk.fileno()] = client # Using fn as key debug('%s connected', addr)