def chown(self, path, uid, gid): MogamiLog.debug("path=%s uid=%d gid=%d" % (path, uid, gid)) try: os.chown(path, uid, gid) ans = 0 except os.error, e: ans = e.errno
def utime(self, path, times): MogamiLog.debug("path = %s, times = %s" % (path, str(times))) try: os.utime(path, times) ans = 0 except os.error, e: ans = e.errno
def rmdir(self, path): MogamiLog.debug("path=%s" % (path)) try: os.rmdir(path) ans = 0 except os.error, e: ans = e.errno
def __init__(self, fsize, dest, data_path, flag, *mode): MogamiFile.__init__(self, fsize) self.remote = True self.dest = dest self.data_path = data_path self.flag = flag self.mode = mode self.prenum = 1 # calculation of the number of blocks self.blnum = self.fsize / conf.blsize if self.fsize % conf.blsize != 0: self.blnum += 1 # initialization of read buffer self.r_data = tuple([MogamiBlock() for i in range(self.blnum + 1)]) self.r_buflock = threading.Lock() MogamiLog.debug("create r_data 0-%d block" % (len(self.r_data)-1)) # initialization of write buffer self.w_list = [] self.w_data = cStringIO.StringIO() self.w_buflock = threading.Lock() self.w_len = 0 # for buffer of dirty data self.dirty_dict = {}
def chmod(self, path, mode): MogamiLog.debug("path = %s w/ mode %s" % (path, oct(mode))) try: os.chmod(path, mode) ans = 0 except os.error, e: ans = e.errno
def release(self, flags): if conf.ap == True: start_t = time.time() MogamiLog.debug("** release **") fsize = self.mogami_file.release(flags) ans = m_channel.release_req(self.metafd, fsize) # delete file size cache if self.path in file_size_dict: del file_size_dict[self.path] if conf.ap == True: # prepare data to tell access pattern myname = m_channel.getmyname() (read_data, write_data) = self.access_pattern.mk_form_data() pid = self.access_pattern.pid cmd_args = self.access_pattern.cmd_args path = self.access_pattern.path end_t = time.time() self.took_time += end_t - start_t if cmd_args != None: # process access pattern history (may be inserted to queue) file_access_queue.put((cmd_args, pid, path, myname, self.took_time, self.created, read_data, write_data)) return 0
def rename(self, oldpath, newpath): MogamiLog.debug(oldpath + ' -> ' + newpath) try: os.rename(oldpath, newpath) ans = 0 except os.error, e: ans = e.errno
def release(self, flags): if conf.ap == True: start_t = time.time() MogamiLog.debug("** release **") fsize = self.mogami_file.release(flags) ans = m_channel.release_req(self.metafd, fsize) # delete file size cache if self.path in file_size_dict: del file_size_dict[self.path] if conf.ap == True: # prepare data to tell access pattern myname = m_channel.getmyname() (read_data, write_data) = self.access_pattern.mk_form_data() pid = self.access_pattern.pid cmd_args = self.access_pattern.cmd_args path = self.access_pattern.path end_t = time.time() self.took_time += end_t - start_t if cmd_args != None: # process access pattern history (may be inserted to queue) file_access_queue.put( (cmd_args, pid, path, myname, self.took_time, self.created, read_data, write_data)) return 0
def __init__(self, metaaddr, rootpath, mogami_dir): """This is the function of MogamiMeta's init. @param metaaddr ip address or hostname of metadata server @param rootpath path of directory to store data into @param mogami_dir path of mogami's root directory """ # basic information of metadata server self.metaaddr = metaaddr self.rootpath = os.path.abspath(rootpath) self.mogami_dir = mogami_dir # check directory for data files assert os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK) # Initialization of Log. MogamiLog.init("data", conf.data_loglevel) MogamiLog.info("Start initialization...") MogamiLog.debug("rootpath = " + self.rootpath) # At first, connect to metadata server and send request to attend. self.m_channel = Channel.MogamiChanneltoMeta() self.m_channel.connect(self.metaaddr) MogamiLog.debug("Success in creating connection to metadata server") self.m_channel.dataadd_req(self.rootpath) MogamiLog.debug("Init complete!!")
def symlink(self, frompath, topath): MogamiLog.debug("frompath = %s, topath = %s" % (frompath, topath)) try: os.symlink(frompath, topath) ans = 0 except os.error, e: ans = e.errno
def mkdir(self, path, mode): MogamiLog.debug("path = %s mode = %o" % (path, mode)) try: os.mkdir(path, mode) ans = 0 except os.error, e: ans = e.errno
def __init__(self, fsize, dest, data_path, flag, *mode): MogamiFile.__init__(self, fsize) self.remote = True self.dest = dest self.data_path = data_path self.flag = flag self.mode = mode self.prenum = 1 # calculation of the number of blocks self.blnum = self.fsize / conf.blsize if self.fsize % conf.blsize != 0: self.blnum += 1 # initialization of read buffer self.r_data = tuple([MogamiBlock() for i in range(self.blnum + 1)]) self.r_buflock = threading.Lock() MogamiLog.debug("create r_data 0-%d block" % (len(self.r_data) - 1)) # initialization of write buffer self.w_list = [] self.w_data = cStringIO.StringIO() self.w_buflock = threading.Lock() self.w_len = 0 # for buffer of dirty data self.dirty_dict = {}
def read(self, length, offset): """read handler. return strings read from file @param length request size of read @param offset offset of read request """ MogamiLog.debug("**read offset=%d, length=%d" % (offset, length)) # check access pattern #self.access_pattern.check_mode(offset, length) #self.access_pattern.change_info(offset, ret_str.tell(), # last_readbl) """ if conf.prefetch == True: if last_readbl != self.access_pattern.last_bl: #MogamiLog.debug("read: prenum %d \n" % (self.prenum)) if conf.force_prenum == True: self.prenum = conf.prenum blnum_list = [] prereq_list = self.access_pattern.return_need_block( (last_readbl + 1) * 1024 * 1024, self.prenum, self.blnum) for prereq in prereq_list: if prereq > self.blnum: break if self.bldata[prereq].state == 0: blnum_list.append(prereq) if len(blnum_list) != 0: self.request_prefetch(blnum_list) self.calc_time += end_t - start_t""" return self.mogami_file.read(length, offset)
def __init__(self, path, flag, *mode): """Initializer called when opened. @param path file path @param flag flags with open(2) @param *mode file open mode (may not be specified) """ MogamiLog.debug("** open ** path = %s, flag = %s, mode = %s" % (path, str(flag), str(mode))) if conf.ap == True: start_t = time.time() # parse argurments self.path = path self.flag = flag self.mode = mode (ans, dest, self.metafd, data_path, self.fsize, self.created) = m_channel.open_req(path, flag, *mode) if ans != 0: # error on metadata server e = IOError() e.errno = ans raise e if dest == 'self': self.mogami_file = FileManager.MogamiLocalFile( self.fsize, data_path, flag, *mode) else: self.mogami_file = FileManager.MogamiRemoteFile( self.fsize, dest, data_path, flag, *mode) ans = self.mogami_file.create_connections(channels) if ans != 0: MogamiLog.error("open error !!") e = IOError() e.errno = ans raise e # register file size to file size dictionary file_size_dict[path] = self.fsize if conf.ap == True: """Get Id list to know pid. list = {gid: pid: uid} And then get the command from pid. """ try: id_list = self.GetContext() pid = id_list['pid'] f = open(os.path.join("/proc", str(pid), "cmdline"), 'r') cmd_args = f.read().rsplit('\x00')[:-1] except Exception, e: # with any error, pass this part of process cmd_args = None pid = -1 self.access_pattern = FileManager.MogamiAccessPattern( path, cmd_args, pid) end_t = time.time() self.took_time = end_t - start_t
def unlink(self, path): """unlink handler. @param path path name to unlink """ MogamiLog.debug("** unlink ** path = %s" % (path, )) ans = m_channel.unlink_req(path) return -ans
def data_add(self, ip, rootpath): self.sysinfo.add_data_server(ip, rootpath) print "add data server IP:", ip print "Now %d data servers are." % len(self.sysinfo.data_list) MogamiLog.info("delete data server IP: %s" % ip) MogamiLog.info("Now there are %d data servers." % len(self.sysinfo.data_list))
def unlink(self, path): MogamiLog.debug("path = %s" % path) if os.path.isfile(path): try: (dest, data_path, fsize) = meta_file_info(path) self.sysinfo.add_delfile(dest, data_path) except Exception, e: MogamiLog.error("cannot remove file contents of %s", path)
def utime(self, path, times): print "** utime **" MogamiLog.debug("path = %s, times = %s" % (path, str(times))) try: os.utime(path, times) ans = 0 except os.error, e: ans = e.errno
def data_del(self, ip): ret = self.sysinfo.remove_data_server(ip) if ret == True: print "delete data server IP:", ip print "Now %d data servers are." % len(self.sysinfo.data_list) MogamiLog.info("delete data server IP: %s" % ip) MogamiLog.info("Now there are %d data servers." % len(self.sysinfo.data_list))
def register_ramfiles(self, add_file_list): """register files in list to files to manage on memory @param add_file_list """ ramfile_list.extend(add_file_list) MogamiLog.debug("** register ramfiles **") MogamiLog.debug("add files = " + str(add_file_list))
def access(self, path, mode): MogamiLog.debug("path = %s" % (path)) try: if os.access(path, mode) == True: ans = 0 else: ans = errno.EACCES except os.error, e: ans = e.errno
def readlink(self, path): print "path = " + path MogamiLog.debug("path = %s" % path) try: result = os.readlink(path) ans = 0 except os.error, e: ans = e.errno result = None
def access(self, path, mode): """access handler. @param path path to access @param mode mode to access @return 0 on success, errno on error """ MogamiLog.debug("** access **" + path + str(mode)) ans = m_channel.access_req(path, mode) return -ans
def chown(self, path, uid, gid): """chown handler. @param path path to change owner of @param uid user id of new owner @param gid group id of new owner """ MogamiLog.debug('** chown ** ' + path + str(uid) + str(gid)) ans = m_channel.chown_req(path, uid, gid) return -ans
def truncate(self, path, length): MogamiLog.debug("path = %s. length = %d" % (path, length)) try: f = open(path, "r+") f.truncate(length) f.close() ans = 0 except Exception, e: MogamiLog.error("have an error (%s)" % (e)) ans = e.errno
def mkdir(self, path, mode): """mkdir handler. @param path directory path to mkdir @param mode permission of the directory to create @return 0 on success, errno on error """ MogamiLog.debug("** mkdir **" + path + str(mode)) ans = m_channel.mkdir_req(path, mode) return -ans
def release(self, flags): MogamiLog.debug("** release **") fsize = self.mogami_file.release(flags) ans = m_channel.release_req(self.metafd, fsize) # delete file size cache if self.path in file_size_dict: del file_size_dict[self.path] return 0
def symlink(self, frompath, topath): """symlink handler. @param frompath @param topath """ MogamiLog.debug("** symlink ** frompath = %s, topath = %s" % (frompath, topath)) ans = m_channel.symlink_req(frompath, topath) return -ans
def chmod(self, path, mode): """chmod handler. @param path path to change permission of @param mode permission to change """ MogamiLog.debug("** chmod ** path = %s, mode = %s" % (path, oct(mode))) ans = m_channel.chmod_req(path, mode) if ans != 0: return -ans
def open(self, path, flag, *mode): start_t = time.time() MogamiLog.debug("path = %s, flag = %s, mode = %s" % (path, str(flag), str(mode))) flag = flag & ~os.O_EXCL try: fd = os.open(path, flag, *mode) ans = 0 except Exception, e: fd = None ans = e.errno
def rename(self, oldpath, newpath): """rename handler. @param oldpath original path name before rename @param newpath new path name after rename """ MogamiLog.debug("** rename ** oldpath = %s, newpath = %s" % (oldpath, newpath)) ans = m_channel.rename_req(oldpath, newpath) if ans != 0: return -ans
def readdir(self, path, offset): MogamiLog.debug("** readdir ** path = %s, offset = %s" % (path, str(offset))) (ans, contents) = m_channel.readdir_req(path, offset) l = ['.', '..'] if ans == 0: l.extend(contents) return [fuse.Direntry(ent) for ent in l] else: return -ans
def getattr(self, path): MogamiLog.debug("path = %s" % path) # get result of stat (ans and st) try: st = os.lstat(path) ans = 0 except os.error, e: MogamiLog.debug("stat error!") ans = e.errno st = None
def request_prefetch(self, blnum_list): """send request of prefetch. @param blnum_list the list of block numbers to prefetch """ MogamiLog.debug("** prefetch ** required blocks = %s" % (str(blnum_list))) # send request to data server self.p_channel.prefetch_req(self.datafd, blnum_list) with self.r_buflock: for blnum in blnum_list: self.r_data[blnum].state = 1
def remove_data_server(self, ip): """remove a data server from Mogami system @param ip ip address of node to remove @return 0 with success, -1 with error """ try: self.data_list.remove(ip) del self.data_rootpath[ip] return True except Exception, e: MogamiLog.error("cannot find %s from data servers" % (ip)) return False