def __init__(self, path, flag, *mode): """Initializer called when opened. @param path file path @param flag flags with open(2) @param *mode file open mode (may not be specified) """ MogamiLog.debug("** open ** path = %s, flag = %s, mode = %s" % (path, str(flag), str(mode))) if conf.ap == True: start_t = time.time() # parse argurments self.path = path self.flag = flag self.mode = mode (ans, dest, self.metafd, data_path, self.fsize, self.created) = m_channel.open_req(path, flag, *mode) if ans != 0: # error on metadata server e = IOError() e.errno = ans raise e if dest == 'self': self.mogami_file = FileManager.MogamiLocalFile( self.fsize, data_path, flag, *mode) else: self.mogami_file = FileManager.MogamiRemoteFile( self.fsize, dest, data_path, flag, *mode) ans = self.mogami_file.create_connections(channels) if ans != 0: MogamiLog.error("open error !!") e = IOError() e.errno = ans raise e # register file size to file size dictionary file_size_dict[path] = self.fsize if conf.ap == True: """Get Id list to know pid. list = {gid: pid: uid} And then get the command from pid. """ try: id_list = self.GetContext() pid = id_list['pid'] f = open(os.path.join("/proc", str(pid), "cmdline"), 'r') cmd_args = f.read().rsplit('\x00')[:-1] except Exception, e: # with any error, pass this part of process cmd_args = None pid = -1 self.access_pattern = FileManager.MogamiAccessPattern( path, cmd_args, pid) end_t = time.time() self.took_time = end_t - start_t
def unlink(self, path): MogamiLog.debug("path = %s" % path) if os.path.isfile(path): try: (dest, data_path, fsize) = meta_file_info(path) self.sysinfo.add_delfile(dest, data_path) except Exception, e: MogamiLog.error("cannot remove file contents of %s", path)
def truncate(self, path, length): MogamiLog.debug("path = %s. length = %d" % (path, length)) try: f = open(path, "r+") f.truncate(length) f.close() ans = 0 except Exception, e: MogamiLog.error("have an error (%s)" % (e)) ans = e.errno
def remove_data_server(self, ip): """remove a data server from Mogami system @param ip ip address of node to remove @return 0 with success, -1 with error """ try: self.data_list.remove(ip) del self.data_rootpath[ip] return True except Exception, e: MogamiLog.error("cannot find %s from data servers" % (ip)) return False
def read(self, fd, blnum): MogamiLog.debug("fd = %d, bl_num = %d" % (fd, blnum)) sendbuf = "" try: os.lseek(fd, blnum * conf.blsize, os.SEEK_SET) buf = cStringIO.StringIO() readlen = 0 while readlen < conf.blsize - 1: os.lseek(fd, blnum * conf.blsize + readlen, os.SEEK_SET) tmpbuf = os.read(fd, conf.blsize - readlen) if tmpbuf == '': # end of file break buf.write(tmpbuf) readlen += len(tmpbuf) sendbuf = buf.getvalue() ans = 0 except Exception, e: MogamiLog.error("read have an error (%s)" % (e)) ans = e.errno
def flush(self, fd, listlen, datalen): MogamiLog.debug("fd=%d, listlen=%d, datalen=%d" % (fd, listlen, datalen)) (write_list, buf) = self.c_channel.flush_recv_data(listlen, datalen) if len(write_list) != 0: write_len = 0 for wd in write_list: try: ans = 0 os.lseek(fd, wd[0], os.SEEK_SET) ret = os.write(fd, buf[write:write + wd[1]]) write_len += ret if ret != wd[1]: MogamiLog.error("write length error !!") break MogamiLog.debug("write from offset %d (result %d)" % (wd[0], ret)) except Exception, e: ans = e.errno break self.c_channel.flush_answer(ans, write_len)
def prefetch(self, fd, blnum_list): for blnum in blnum_list: try: sendbuf = "" start_t = time.time() MogamiLog.debug("fd = %d, blnum = %d" % (fd, bl_num)) os.lseek(fd, bl_num * conf.blsize, os.SEEK_SET) buf = cStringIO.StringIO() readlen = 0 while readlen < conf.blsize - 1: os.lseek(fd, bl_num * conf.blsize + readlen, os.SEEK_SET) tmpbuf = os.read(fd, conf.blsize - readlen) if tmpbuf == '': # end of file break buf.write(tmpbuf) readlen += len(tmpbuf) sendbuf = buf.getvalue() end_t = time.time() # send data read from file (only in case w/o error) self.c_channel.data_send(0, blnum, len(sendbuf), sendbuf) except Exception, e: MogamiLog.error("Prefetch Error!! with %d-th block" % (blnum))
def run(self, ): while True: req = self.c_channel.recv_request() if req == None: MogamiLog.debug("Connection closed") self.c_channel.finalize() break if req[0] == Channel.REQ_GETATTR: MogamiLog.debug("** getattr **") self.getattr(self.rootpath + req[1]) elif req[0] == Channel.REQ_READDIR: MogamiLog.debug("** readdir **") self.readdir(self.rootpath + req[1]) elif req[0] == Channel.REQ_ACCESS: MogamiLog.debug("** access **") self.access(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_MKDIR: MogamiLog.debug("** mkdir **") self.mkdir(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_RMDIR: MogamiLog.debug("** rmdir **") self.rmdir(self.rootpath + req[1]) elif req[0] == Channel.REQ_UNLINK: MogamiLog.debug("** unlink **") self.unlink(self.rootpath + req[1]) elif req[0] == Channel.REQ_RENAME: MogamiLog.debug("** rename **") self.rename(self.rootpath + req[1], self.rootpath + req[2]) elif req[0] == Channel.REQ_MKNOD: MogamiLog.debug("** mknod **") self.mknod(self.rootpath + req[1], req[2], req[3]) elif req[0] == Channel.REQ_CHMOD: MogamiLog.debug("** chmod **") self.chmod(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_CHOWN: MogamiLog.debug("** chown **") self.chown(self.rootpath + req[1], req[2], req[3]) elif req[0] == Channel.REQ_LINK: self.link(self.rootpath + req[1], self.rootpath + req[2]) elif req[0] == Channel.REQ_SYMLINK: self.symlink(req[1], self.rootpath + req[2]) elif req[0] == Channel.REQ_READLINK: self.readlink(self.rootpath + req[1]) elif req[0] == Channel.REQ_TRUNCATE: self.truncate(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_UTIME: self.utime(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_FSYNC: self.fsync(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_OPEN: self.open(self.rootpath + req[1], req[2], req[3]) elif req[0] == Channel.REQ_RELEASE: MogamiLog.debug("** release **") self.release(req[1], req[2]) elif req[0] == Channel.REQ_FGETATTR: self.fgetattr(req[1]) elif req[0] == Channel.REQ_FTRUNCATE: MogamiLog.error("** ftruncate **") elif req[0] == Channel.REQ_DATAADD: MogamiLog.debug("** dataadd **") ip = self.c_channel.getpeername() self.data_add(ip, req[1]) elif req[0] == Channel.REQ_DATADEL: MogamiLog.debug("** datadel **") ip = self.c_channel.getpeername() self.data_del(ip) elif req[0] == Channel.REQ_RAMFILEADD: MogamiLog.debug("** ramfile add **") self.register_ramfiles(req[1]) elif req[0] == Channel.REQ_FILEASK: MogamiLog.debug("** fileask **") self.file_ask(req[1]) else: MogamiLog.error("[error] Unexpected Header") self.c_channel.finalize() break
class MogamiMetaHandler(Daemons.MogamiDaemons): """This is the class for thread created for each client. This handler is run as multithread. """ def __init__(self, client_channel, sysinfo): Daemons.MogamiDaemons.__init__(self) self.sysinfo = sysinfo self.c_channel = client_channel self.rootpath = sysinfo.meta_rootpath def run(self, ): while True: req = self.c_channel.recv_request() if req == None: MogamiLog.debug("Connection closed") self.c_channel.finalize() break if req[0] == Channel.REQ_GETATTR: MogamiLog.debug("** getattr **") self.getattr(self.rootpath + req[1]) elif req[0] == Channel.REQ_READDIR: MogamiLog.debug("** readdir **") self.readdir(self.rootpath + req[1]) elif req[0] == Channel.REQ_ACCESS: MogamiLog.debug("** access **") self.access(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_MKDIR: MogamiLog.debug("** mkdir **") self.mkdir(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_RMDIR: MogamiLog.debug("** rmdir **") self.rmdir(self.rootpath + req[1]) elif req[0] == Channel.REQ_UNLINK: MogamiLog.debug("** unlink **") self.unlink(self.rootpath + req[1]) elif req[0] == Channel.REQ_RENAME: MogamiLog.debug("** rename **") self.rename(self.rootpath + req[1], self.rootpath + req[2]) elif req[0] == Channel.REQ_MKNOD: MogamiLog.debug("** mknod **") self.mknod(self.rootpath + req[1], req[2], req[3]) elif req[0] == Channel.REQ_CHMOD: MogamiLog.debug("** chmod **") self.chmod(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_CHOWN: MogamiLog.debug("** chown **") self.chown(self.rootpath + req[1], req[2], req[3]) elif req[0] == Channel.REQ_LINK: self.link(self.rootpath + req[1], self.rootpath + req[2]) elif req[0] == Channel.REQ_SYMLINK: self.symlink(req[1], self.rootpath + req[2]) elif req[0] == Channel.REQ_READLINK: self.readlink(self.rootpath + req[1]) elif req[0] == Channel.REQ_TRUNCATE: self.truncate(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_UTIME: self.utime(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_FSYNC: self.fsync(self.rootpath + req[1], req[2]) elif req[0] == Channel.REQ_OPEN: self.open(self.rootpath + req[1], req[2], req[3]) elif req[0] == Channel.REQ_RELEASE: MogamiLog.debug("** release **") self.release(req[1], req[2]) elif req[0] == Channel.REQ_FGETATTR: self.fgetattr(req[1]) elif req[0] == Channel.REQ_FTRUNCATE: MogamiLog.error("** ftruncate **") elif req[0] == Channel.REQ_DATAADD: MogamiLog.debug("** dataadd **") ip = self.c_channel.getpeername() self.data_add(ip, req[1]) elif req[0] == Channel.REQ_DATADEL: MogamiLog.debug("** datadel **") ip = self.c_channel.getpeername() self.data_del(ip) elif req[0] == Channel.REQ_RAMFILEADD: MogamiLog.debug("** ramfile add **") self.register_ramfiles(req[1]) elif req[0] == Channel.REQ_FILEASK: MogamiLog.debug("** fileask **") self.file_ask(req[1]) else: MogamiLog.error("[error] Unexpected Header") self.c_channel.finalize() break # MogamiSystem APIs def data_add(self, ip, rootpath): self.sysinfo.add_data_server(ip, rootpath) print "add data server IP:", ip print "Now %d data servers are." % len(self.sysinfo.data_list) MogamiLog.info("delete data server IP: %s" % ip) MogamiLog.info("Now there are %d data servers." % len(self.sysinfo.data_list)) def data_del(self, ip): ret = self.sysinfo.remove_data_server(ip) if ret == True: print "delete data server IP:", ip print "Now %d data servers are." % len(self.sysinfo.data_list) MogamiLog.info("delete data server IP: %s" % ip) MogamiLog.info("Now there are %d data servers." % len(self.sysinfo.data_list)) def register_ramfiles(self, add_file_list): """register files in list to files to manage on memory @param add_file_list """ ramfile_list.extend(add_file_list) MogamiLog.debug("** register ramfiles **") MogamiLog.debug("add files = " + str(add_file_list)) def remove_ramfiles(self, file_list): """not implemented yet.. """ pass # Mogami's actual metadata access APIs def getattr(self, path): MogamiLog.debug("path = %s" % path) # get result of stat (ans and st) try: st = os.lstat(path) ans = 0 except os.error, e: MogamiLog.debug("stat error!") ans = e.errno st = None # get file size if os.path.isfile(path): try: fsize = meta_file_info(path)[2] MogamiLog.debug("%s's size is %d" % (path, fsize)) except Exception, e: fsize = 0 MogamiLog.error("error in meta_file_info")