def utime(self, path, times): DRDFSLog.debug("path = %s, times = %s" % (path, times)) try: os.utime(path, times) senddata = 0 except os.error, e: senddata = e.errno
def release(self, fd, dest, fsize, filename, created): """release handler. @param fd file discripter @param writelen size of data to be written """ os.lseek(fd, 0, os.SEEK_SET) DRDFSLog.debug("fd = " + str(fd)) if created == False: self.repl_info.ReplInfoWhenClose(filename, dest, self.access_info) else: try: buf = os.read(fd, conf.bufsize) except os.error, e: print "OSError in release (%s)" %(e) l = buf.rsplit(',') size = string.atol(l[2]) try: buf = l[0] + ',' + l[1] + ',' + str(fsize) + ',\n' DRDFSLog.debug("write to meta file %s" % buf) os.ftruncate(fd, len(buf)) os.lseek(fd, 0, os.SEEK_SET) os.write(fd, buf) os.fsync(fd) except os.error, e: print "OSError in release (%s)" %(e)
def rename(self, oldpath, newpath): DRDFSLog.debug(oldpath + ' -> ' + newpath) try: os.rename(oldpath, newpath) senddata = 0 except os.error, e: senddata = e.errno
def mkdir(self, path, mode): DRDFSLog.debug('path=%s mode=%o' % (path, mode)) try: os.mkdir(path, mode) senddata = 0 except os.error, e: senddata = e.errno
def rmdir(self, path): DRDFSLog.debug('path=%s' % (path)) try: os.rmdir(path) senddata = 0 except os.error, e: senddata = e.errno
def chmod(self, path, mode): DRDFSLog.debug('path=%s w/ mode %o' % (path, mode)) try: os.chmod(path, mode) senddata = 0 except os.error, e: senddata = e.errno
def chown(self, path, uid, gid): DRDFSLog.debug("path=%s uid=%d gid=%d" % (path, uid, gid)) try: os.chown(path, uid, gid) senddata = 0 except os.error, e: senddata = e.errno
def release(self, fd, dest, fsize, filename, created): """release handler. @param fd file discripter @param writelen size of data to be written """ os.lseek(fd, 0, os.SEEK_SET) DRDFSLog.debug("fd = " + str(fd)) if created == False: self.repl_info.ReplInfoWhenClose(filename, dest, self.access_info) else: try: buf = os.read(fd, conf.bufsize) except os.error, e: print "OSError in release (%s)" % (e) l = buf.rsplit(',') size = string.atol(l[2]) try: buf = l[0] + ',' + l[1] + ',' + str(fsize) + ',\n' DRDFSLog.debug("write to meta file %s" % buf) os.ftruncate(fd, len(buf)) os.lseek(fd, 0, os.SEEK_SET) os.write(fd, buf) os.fsync(fd) except os.error, e: print "OSError in release (%s)" % (e)
def utime(self, path, times): DRDFSLog.debug("** utime ** path = %s, times = %s" % (path, str(times))) senddata = ['utime', path, times] ans = m_channel.send_recv_flow(senddata) return -ans
def access(self, path, mode): DRDFSLog.debug("** access **" + path + str(mode)) senddata = ["access", path, mode] ans = m_channel.send_recv_flow(senddata) if ans != True: return -errno.EACCES return 0
def readdir(self, path): DRDFSLog.debug('path=%s' % (path)) try: l = os.listdir(path) senddata = [0, l] except os.error, e: senddata = [e.errno, "null"] print 'error!'
def access(self, path, mode): DRDFSLog.debug("** access **" + path + str(mode)) senddata = ['access', path, mode] ans = m_channel.send_recv_flow(senddata) if ans != True: return -errno.EACCES return 0
def getattr(self, path): DRDFSLog.debug('path = ' + path) try: st = os.lstat(path) except os.error, e: DRDFSLog.debug("stat error!") senddata = [e.errno, "null"] self.c_channel.send_header(senddata) return
def run(self,): while True: daemons_alive = threading.enumerate() for d in self.daemons: if d not in daemons_alive: d.join() DRDFSLog.debug("** join thread **") self.daemons.remove(d) time.sleep(3)
def run(self, ): while True: daemons_alive = threading.enumerate() for d in self.daemons: if d not in daemons_alive: d.join() DRDFSLog.debug("** join thread **") self.daemons.remove(d) time.sleep(3)
def access(self, path, mode): DRDFSLog.debug('path=%s' % (path)) try: if os.access(path, mode) == True: senddata = True else: senddata = False except os.error, e: senddata = False
def readdir(self, path, offset): DRDFSLog.debug("** readdir **" + path + str(offset)) senddata = ["readdir", path] ans = m_channel.send_recv_flow(senddata) l = [".", ".."] if ans[0] == 0: l.extend(ans[1]) return [fuse.Direntry(ent) for ent in l] else: return -ans[0]
def readdir(self, path, offset): DRDFSLog.debug("** readdir **" + path + str(offset)) senddata = ['readdir', path] ans = m_channel.send_recv_flow(senddata) l = ['.', '..'] if ans[0] == 0: l.extend(ans[1]) return [fuse.Direntry(ent) for ent in l] else: return -ans[0]
def open(self, path, flag, *mode): DRDFSLog.debug("path = %s, flag = %s, mode = %s" % (path, str(flag), str(mode))) """I should fill in this function. Flow of creating new file should be written. """ flag = flag & ~os.O_EXCL try: fd = os.open(os.path.join(self.rootpath, path), flag, *mode) senddata = [0, fd] except Exception, e: senddata = [e.errno, 0]
def getattr(self, path): DRDFSLog.debug("** getattr ** path = %s" % path) senddata = ['getattr', path] ans = m_channel.send_recv_flow(senddata) if ans[0] != 0: return -ans[0] else: st = tips.DRDFSStat() st.load(ans[1]) if ans[2] >= 0: st.chsize(ans[2]) return st
def getattr(self, path): DRDFSLog.debug("** getattr ** path = %s" % path) senddata = ["getattr", path] ans = m_channel.send_recv_flow(senddata) if ans[0] != 0: return -ans[0] else: st = tips.DRDFSStat() st.load(ans[1]) if ans[2] >= 0: st.chsize(ans[2]) return st
def read(self, length, offset): DRDFSLog.debug("** read ** offset %d, length =%d" % (offset, length)) requestl = self.cal_bl(offset, length) # prepare buffer to return ret_str = cStringIO.StringIO() DRDFSLog.debug("requestl = %s, with offset: %d, length: %d" % (str(requestl), offset, length)) for req in requestl: reqbl = req[0] buf = "" # buffer for block[reqbl] last_readbl = reqbl if self.bldata[reqbl].state == 2: buf = self.bldata[reqbl].buf elif self.bldata[reqbl].state == 1: DRDFSLog.debug("Waiting recv data %d block" % reqbl) while self.bldata[reqbl].state == 1: time.sleep(0.01) buf = self.bldata[reqbl].buf else: ret = self.request_block(reqbl) if ret != 0: DRDFSLog.error("read error! (%d)" % ret) return ret while self.bldata[reqbl].state == 1: time.sleep(0.01) with self.r_buflock: buf = self.bldata[reqbl].buf # write strings to return and if reach EOF, break ret_str.write(buf[req[1] : req[2]]) if len(buf) < conf.blsize: break # end of file return ret_str.getvalue()
def release(self, flags): DRDFSLog.debug("** release") if self.writelen == 0: pass else: self._fflush() senddata = ["release", self.datafd] ans = self.d_channel.send_recv_flow(senddata) self.bldata = None self.timedic = None senddata = ["release", self.metafd, self.dist, ans, self.path, self.created] ans = m_channel.send_recv_flow(senddata) return 0
def unlink(self, path): DRDFSLog.debug(path) if os.path.isfile(path): f = open(path, 'r') while True: buf = f.readline() if buf == '': break l = buf.rsplit(',') self.delfiles_q.put((l[0], l[1])) self.access_info.del_file(path) f.close() try: os.unlink(path) senddata = 0 except os.error, e: senddata = e.errno
def _fflush(self,): with self.w_buflock: buf = cPickle.dumps(self.writelist) data = self.writedata.getvalue() senddata = ["flush", self.datafd, len(buf), len(data)] DRDFSLog.debug("flush: fd=%d, listlen=%d, datalen=%d" % (self.datafd, len(buf), len(data))) with self.d_channel.lock: self.d_channel._send_header(senddata) self.d_channel.sock.sendall(buf) self.d_channel.sock.sendall(data) ans = self.d_channel._recv_header() self.writelen = 0 self.writelist = [] self.writedata = cStringIO.StringIO() self.dirty_dict = {} return ans
def _fflush(self, ): with self.w_buflock: buf = cPickle.dumps(self.writelist) data = self.writedata.getvalue() senddata = ['flush', self.datafd, len(buf), len(data)] DRDFSLog.debug("flush: fd=%d, listlen=%d, datalen=%d" % (self.datafd, len(buf), len(data))) with self.d_channel.lock: self.d_channel._send_header(senddata) self.d_channel.sock.sendall(buf) self.d_channel.sock.sendall(data) ans = self.d_channel._recv_header() self.writelen = 0 self.writelist = [] self.writedata = cStringIO.StringIO() self.dirty_dict = {} return ans
def release(self, flags): DRDFSLog.debug('** release') if self.writelen == 0: pass else: self._fflush() senddata = ['release', self.datafd] ans = self.d_channel.send_recv_flow(senddata) self.bldata = None self.timedic = None senddata = [ 'release', self.metafd, self.dist, ans, self.path, self.created ] ans = m_channel.send_recv_flow(senddata) return 0
def truncate(self, path, len): senddata = ["truncate", path, len] DRDFSLog.debug("*** truncate ***" + path + str(len)) ans = m_channel.send_recv_flow(senddata) if ans[0] != 0: return -ans[0] ans[1] = filedist_info_list DRDFSLog.debug("send request to data server: path=%s, len=%d" % (path, len)) for filedist_info in filedist_info_list: dist = filedist_info[0] filename = filedist_info[1] senddata = ["truncate", filename, len] ch = channel.DRDFSChannel() ch.connect(dist, conf.dataport) ans = ch.send_recv_flow(senddata) ch.brk_channel() if ans != 0: return -ans return 0
def truncate(self, path, len): senddata = ['truncate', path, len] DRDFSLog.debug('*** truncate ***' + path + str(len)) ans = m_channel.send_recv_flow(senddata) if ans[0] != 0: return -ans[0] ans[1] = filedist_info_list DRDFSLog.debug("send request to data server: path=%s, len=%d" % (path, len)) for filedist_info in filedist_info_list: dist = filedist_info[0] filename = filedist_info[1] senddata = ['truncate', filename, len] ch = channel.DRDFSChannel() ch.connect(dist, conf.dataport) ans = ch.send_recv_flow(senddata) ch.brk_channel() if ans != 0: return -ans return 0
def truncate(self, path, len): """truncate handler. @param path file path @param len length of output file """ DRDFSLog.debug("path = %s, len = %d" % (path, len)) try: f = open(path, 'r+') filedist_info_list = [] while True: # read original data buf = f.readline() if buf == '': # EOF break l = buf.rsplit(',') if len(l) == 3: filedist_info_list.append((l[0], l[1])) elif len(l) == 4: filedist_info_list.append((l[0], l[1], l[3])) else: break f.truncate(0) f.seek(0) for filedist_info in filedist_info_list: dist = filedist_info[0] filename = filedist_info[1] if len(filedist_info) == 2: filesource = '' else: filesource = filedist_info[2] f.write("%s,%s,%d,%s\n" % (dist, filename, len, filesource)) f.close() senddata = [0, filedist_info_list] except IOError, e: senddata = [ e.errno, ]
def request_block(self, blnum): """used when client send a request of block data. """ senddata = ['read', self.datafd, blnum] DRDFSLog.debug("** read %d block" % (blnum)) with self.r_buflock: self.bldata[blnum].state = 1 DRDFSLog.debug("request to data server %d block" % (blnum)) with self.d_channel.lock: self.d_channel._send_header(senddata) ans = self.d_channel._recv_header() if ans == None: return 0 errno = ans[0] blnum = ans[1] size = ans[2] if size == 0: DRDFSLog.debug("*********======") with self.r_buflock: self.bldata[blnum].state = 2 self.bldata[blnum].buf = "" return 0 buf = self.d_channel._recvall(size) if self.bldata != None: with self.r_buflock: self.bldata[blnum].state = 2 self.bldata[blnum].buf = buf return 0
def request_block(self, blnum): """used when client send a request of block data. """ senddata = ["read", self.datafd, blnum] DRDFSLog.debug("** read %d block" % (blnum)) with self.r_buflock: self.bldata[blnum].state = 1 DRDFSLog.debug("request to data server %d block" % (blnum)) with self.d_channel.lock: self.d_channel._send_header(senddata) ans = self.d_channel._recv_header() if ans == None: return 0 errno = ans[0] blnum = ans[1] size = ans[2] if size == 0: DRDFSLog.debug("*********======") with self.r_buflock: self.bldata[blnum].state = 2 self.bldata[blnum].buf = "" return 0 buf = self.d_channel._recvall(size) if self.bldata != None: with self.r_buflock: self.bldata[blnum].state = 2 self.bldata[blnum].buf = buf return 0
def __init__(self, rootpath, dddfs_dir): self.rootpath = os.path.abspath(rootpath) self.dddfs_dir = dddfs_dir """Check directory for meta data files. """ if os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK) == False: sys.exit("%s is not permitted to use. " % (self.rootpath, )) DRDFSLog.init("meta", DRDFSLog.DEBUG) DRDFSLog.info("** DRDFS metadata server init **") DRDFSLog.debug("rootpath = " + self.rootpath) # for replication if conf.replication == True: self.repl_info = ReplicationManager.ReplicationManager() self.cluster_info = cluster.DDDFSNodesInfo( os.path.join(self.dddfs_dir, 'conf', conf.cluster_conf_file)) self.access_info = chooseDataNode.FileAccessInfo() self.delfiles_q = Queue.Queue() self.datalist = [] self.repq = Queue.Queue()
def __init__(self, metaaddr, rootpath, dddfs_dir): """initialize DDDFS's data server daemon """ self.metaaddr = metaaddr self.rootpath = os.path.abspath(rootpath) self.dddfs_dir = dddfs_dir """Check directory for data files. """ assert os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK) """Initialize Log """ DRDFSLog.init("data", DRDFSLog.DEBUG) """At first, connect to metadata server and send request to attend. """ mchannel = channel.DRDFSChannel() mchannel.connect(self.metaaddr, conf.metaport) DRDFSLog.debug("Success in creating connection to metadata server") senddata = ['dataadd', self.rootpath] ans = mchannel.send_recv_flow(senddata) if ans == -1: e = system.DDDFSSystemError() raise e mchannel.brk_channel() DRDFSLog.debug("Init complete!!")
def run(self, ): """Connected from a client """ self.lsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.lsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.lsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.lsock.bind(("0.0.0.0", conf.metaport)) self.lsock.listen(10) DRDFSLog.debug("Listening at the port " + str(conf.metaport)) daemons = [] collector_thread = self.thread_collector(daemons) collector_thread.start() threads_count = 0 delete_files_thread = self.delete_files(self.delfiles_q) delete_files_thread.start() replicator_thread = self.replicator(self.repq, self.access_info, self.rootpath) replicator_thread.start() while True: (csock, address) = self.lsock.accept() c_channel = channel.DRDFSChannel() c_channel.set_channel_from_sock(csock) DRDFSLog.debug("accept connnect from %s" % (str(address[0]))) metad = self.handler(c_channel, self.rootpath, self.delfiles_q, self.datalist, self.repq, self.repl_info, self.access_info, self.cluster_info) threads_count += 1 metad.start() daemons.append(metad) DRDFSLog.debug("Created thread name = " + metad.getName())
def open(self, path, flag, mode): """open handler. @param path file path @param flag flags for open(2) @param mode open mode (may be empty tuple): actual value is mode[0] """ if os.access(self.rootpath + path, os.F_OK) == True: """When the required file exist... """ try: DRDFSLog.debug("!!find the file %s w/ %o" % (path, flag)) created = False fd = 0 if mode: fd = os.open(self.rootpath + path, os.O_RDWR, mode[0]) else: fd = os.open(self.rootpath + path, os.O_RDWR) DRDFSLog.debug("fd = %d" % (fd)) data_buf = "" while True: buf = os.read(fd, conf.bufsize) if buf == '': break data_buf += buf lines = data_buf.rsplit('\n') data_nodes = [] filename = '' for line in lines: l = line.rsplit(',') if len(l) < 3: break data_nodes.append(l[0]) # e.g.) "127.0.0.1" filename = l[1] size = string.atol(l[2]) data_num = len(lines) peername = self.c_channel.sock.getpeername()[0] print path dest = chooseDataNode.ChooseDataNode( path, peername, data_nodes, self.access_info, self.cluster_info) senddata = [0, dest, fd, size, filename, created] self.access_info.add_load(path, dest) except os.error, e: DRDFSLog.debug("!!find the file but error for %s (%s)" % (path, e)) senddata = [e.errno, None, None, None, None, None] self.c_channel.send_header(senddata) # do replication (if need) repl_dict = self.repl_info.ReplInfoWhenOpen( path, data_nodes, self.access_info, self.cluster_info) if repl_dict != None: # add a replication task if conf.replication == True: # select distination in replication self.repq.put((repl_dict['from'], filename, repl_dict['to'], path)) print "task ***"
def open(self, path, flag, mode): """open handler. @param path file path @param flag flags for open(2) @param mode open mode (may be empty tuple): actual value is mode[0] """ if os.access(self.rootpath + path, os.F_OK) == True: """When the required file exist... """ try: DRDFSLog.debug("!!find the file %s w/ %o" % (path, flag)) created = False fd = 0 if mode: fd = os.open(self.rootpath + path, os.O_RDWR, mode[0]) else: fd = os.open(self.rootpath + path, os.O_RDWR) DRDFSLog.debug("fd = %d" % (fd)) data_buf = "" while True: buf = os.read(fd, conf.bufsize) if buf == '': break data_buf += buf lines = data_buf.rsplit('\n') data_nodes = [] filename = '' for line in lines: l = line.rsplit(',') if len(l) < 3: break data_nodes.append(l[0]) # e.g.) "127.0.0.1" filename = l[1] size = string.atol(l[2]) data_num = len(lines) peername = self.c_channel.sock.getpeername()[0] print path dest = chooseDataNode.ChooseDataNode(path, peername, data_nodes, self.access_info, self.cluster_info) senddata = [0, dest, fd, size, filename, created] self.access_info.add_load(path, dest) except os.error, e: DRDFSLog.debug("!!find the file but error for %s (%s)" % (path, e)) senddata = [e.errno, None, None, None, None, None] self.c_channel.send_header(senddata) # do replication (if need) repl_dict = self.repl_info.ReplInfoWhenOpen(path, data_nodes, self.access_info, self.cluster_info) if repl_dict != None: # add a replication task if conf.replication == True: # select distination in replication self.repq.put((repl_dict['from'], filename, repl_dict['to'], path)) print "task ***"
def truncate(self, path, len): """truncate handler. @param path file path @param len length of output file """ DRDFSLog.debug("path = %s, len = %d" % (path, len)) try: f = open(path, 'r+') filedist_info_list = [] while True: # read original data buf = f.readline() if buf == '': # EOF break l = buf.rsplit(',') if len(l) == 3: filedist_info_list.append((l[0], l[1])) elif len(l) == 4: filedist_info_list.append((l[0], l[1], l[3])) else: break f.truncate(0) f.seek(0) for filedist_info in filedist_info_list: dist = filedist_info[0] filename = filedist_info[1] if len(filedist_info) == 2: filesource = '' else: filesource = filedist_info[2] f.write("%s,%s,%d,%s\n" % (dist, filename, len, filesource)) f.close() senddata = [0, filedist_info_list] except IOError, e: senddata = [e.errno, ]
def fsinit(self): """Called before fs.main() called. """ DRDFSLog.info("** DRDFS FS init **") DRDFSLog.debug("Success in creating connection to metadata server") DRDFSLog.debug("Init complete!!") collector_thread = self.thread_collector(daemons) collector_thread.start() threads_count = 0
def read(self, length, offset): DRDFSLog.debug("** read ** offset %d, length =%d" % (offset, length)) requestl = self.cal_bl(offset, length) # prepare buffer to return ret_str = cStringIO.StringIO() DRDFSLog.debug("requestl = %s, with offset: %d, length: %d" % (str(requestl), offset, length)) for req in requestl: reqbl = req[0] buf = "" # buffer for block[reqbl] last_readbl = reqbl if self.bldata[reqbl].state == 2: buf = self.bldata[reqbl].buf elif self.bldata[reqbl].state == 1: DRDFSLog.debug("Waiting recv data %d block" % reqbl) while self.bldata[reqbl].state == 1: time.sleep(0.01) buf = self.bldata[reqbl].buf else: ret = self.request_block(reqbl) if ret != 0: DRDFSLog.error("read error! (%d)" % ret) return ret while self.bldata[reqbl].state == 1: time.sleep(0.01) with self.r_buflock: buf = self.bldata[reqbl].buf # write strings to return and if reach EOF, break ret_str.write(buf[req[1]:req[2]]) if len(buf) < conf.blsize: break # end of file return ret_str.getvalue()
def flush(self, fd, listlen, datalen): DRDFSLog.debug("flush: fd=%d, listlen=%d, datalen=%d" % (fd, listlen, datalen)) buf = self.c_channel._recvall(listlen) writelist = cPickle.loads(buf) buf = self.c_channel._recvall(datalen) write = 0 for wd in writelist: try: os.lseek(fd, wd[0], os.SEEK_SET) res = os.write(fd, buf[write:write+wd[1]]) write += res if res != wd[1]: DRDFSLog.error("write length error !!") DRDFSLog.debug("write from offset %d (result %d)" % (wd[0], res)) except Exception, e: senddata = -e.errno
def flush(self, fd, listlen, datalen): DRDFSLog.debug("flush: fd=%d, listlen=%d, datalen=%d" % (fd, listlen, datalen)) buf = self.c_channel._recvall(listlen) writelist = cPickle.loads(buf) buf = self.c_channel._recvall(datalen) write = 0 for wd in writelist: try: os.lseek(fd, wd[0], os.SEEK_SET) res = os.write(fd, buf[write:write + wd[1]]) write += res if res != wd[1]: DRDFSLog.error("write length error !!") DRDFSLog.debug("write from offset %d (result %d)" % (wd[0], res)) except Exception, e: senddata = -e.errno
def read(self, fd, bl_num): DRDFSLog.debug("fd = %d, bl_num = %d" % (fd, bl_num)) try: os.lseek(fd, bl_num * conf.blsize, os.SEEK_SET) buf = cStringIO.StringIO() readlen = 0 while readlen < conf.blsize - 1: os.lseek(fd, bl_num * conf.blsize + readlen, os.SEEK_SET) tmpbuf = os.read(fd, conf.blsize - readlen) if tmpbuf == '': # end of file break buf.write(tmpbuf) readlen += len(tmpbuf) sendbuf = buf.getvalue() DRDFSLog.debug("read from file offset %d len %d (result %d)" % (bl_num * conf.blsize, conf.blsize, len(sendbuf))) senddata = [0, bl_num, len(sendbuf)] except Exception, e: DRDFSLog.error("read have an error (%s)" % (e)) senddata = [e.errno, 'null', 0, 0]
def __init__(self, rootpath, dddfs_dir): self.rootpath = os.path.abspath(rootpath) self.dddfs_dir = dddfs_dir """Check directory for meta data files. """ if os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK) == False: sys.exit("%s is not permitted to use. " % (self.rootpath, )) DRDFSLog.init("meta", DRDFSLog.DEBUG) DRDFSLog.info("** DRDFS metadata server init **") DRDFSLog.debug("rootpath = " + self.rootpath) # for replication self.repl_info = ReplicationManager.ReplicationManager() self.cluster_info = cluster.DDDFSNodesInfo( os.path.join(self.dddfs_dir, 'conf', conf.cluster_conf_file)) self.access_info = chooseDataNode.FileAccessInfo() self.delfiles_q = Queue.Queue() self.datalist = [] self.repq = Queue.Queue()
def read(self, fd, bl_num): DRDFSLog.debug("fd = %d, bl_num = %d" % (fd, bl_num)) try: os.lseek(fd, bl_num * conf.blsize, os.SEEK_SET) buf = cStringIO.StringIO() readlen = 0 while readlen < conf.blsize - 1: os.lseek(fd, bl_num * conf.blsize + readlen, os.SEEK_SET) tmpbuf = os.read(fd, conf.blsize - readlen) if tmpbuf == '': # end of file break buf.write(tmpbuf) readlen += len(tmpbuf) sendbuf = buf.getvalue() DRDFSLog.debug( "read from file offset %d len %d (result %d)" % (bl_num * conf.blsize, conf.blsize, len(sendbuf))) senddata = [0, bl_num, len(sendbuf)] except Exception, e: DRDFSLog.error("read have an error (%s)" % (e)) senddata = [e.errno, 'null', 0, 0]
def run(self, ): """Connected from a client """ self.lsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.lsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.lsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.lsock.bind(("0.0.0.0", conf.metaport)) self.lsock.listen(10) DRDFSLog.debug("Listening at the port " + str(conf.metaport)) daemons = [] collector_thread = self.thread_collector(daemons) collector_thread.start() threads_count = 0 delete_files_thread = self.delete_files(self.delfiles_q) delete_files_thread.start() replicator_thread = self.replicator(self.repq, self.access_info, self.rootpath) replicator_thread.start() delete_replica_thread = self.del_repl(self.repl_info, self.delfiles_q, self.rootpath) delete_replica_thread.start() while True: (csock, address) = self.lsock.accept() c_channel = channel.DRDFSChannel() c_channel.set_channel_from_sock(csock) DRDFSLog.debug("accept connnect from %s" % (str(address[0]))) metad = self.handler(c_channel, self.rootpath, self.delfiles_q, self.datalist, self.repq, self.repl_info, self.access_info, self.cluster_info) threads_count += 1 metad.start() daemons.append(metad) DRDFSLog.debug("Created thread name = " + metad.getName())
def run(self, ): while True: header = self.c_channel.recv_header() if header == None: DRDFSLog.debug("Connection closed") break if header[0] == 'open': DRDFSLog.debug('** open') self.open(header[1], header[2], *header[3]) elif header[0] == 'create': DRDFSLog.debug('** create') self.create(header[1], header[2], header[3]) elif header[0] == 'read': DRDFSLog.debug('** read') self.read(header[1], header[2]) elif header[0] == 'flush': DRDFSLog.debug('** flush') self.flush(header[1], header[2], header[3]) elif header[0] == 'release': DRDFSLog.debug('** release') self.release(header[1]) break elif header[0] == 'truncate': DRDFSLog.debug('** truncate') self.truncate(header[1], header[2]) elif header[0] == 'ftruncate': DRDFSLog.debug('** ftruncate') elif header[0] == 'close': DRDFSLog.debug('** close') self.c_channel.brk_channel() break elif header[0] == 'filedel': DRDFSLog.debug('** filedel') self.filedel(header[1]) elif header[0] == 'do_repl': DRDFSLog.debug('** do replication **') """header[1]: file path to replicate header[2]: IP address of the node where new replica will be located """ self.replication(header[1], header[2]) elif header[0] == 'recv_repl': DRDFSLog.debug('** recv replication **') """header[1]: file path to replicate header[2]: file replica size """ self.recv_replication(header[1], header[2]) else: DRDFSLog.debug('** this is unexpected header. break!') break