def open(self, path, flag, mode): """open handler. @param path file path @param flag flags for open(2) @param mode open mode (may be empty tuple): actual value is mode[0] """ if self.meta_rep.access(path, os.F_OK) == True: # When the required file exist... try: MogamiLog.debug("!!find the file %s w/ %o" % (path, flag)) affinity = self.c_channel.getpeername() (dest, data_path, fsize) = self.meta_rep.open( path, flag, mode, affinity) # create data to send ans = 0 created = False except Exception, e: MogamiLog.debug("!!find the file but error for %s (%s)" % (path, e)) try: ans = e.errno except Exception, e: ans = errno.ENOENT dest = None fd = None data_path = None fsize = None created = False
def utime(self, path, times): MogamiLog.debug("path = %s, times = %s" % (path, str(times))) try: self.meta_rep.utime(path, times) ans = 0 except os.error, e: ans = e.errno
def fgetattr(self, fd): try: st = os.fstat(fd) senddata = [0, st] except os.error, e: MogamiLog.debug("OSError in fgetattr (%s)" % (e)) senddata = [e.errno, 'null']
def send_delete_request(self, ip, files): MogamiLog.debug("file delete request was sent (%s -> %s)" % (ip, str(files))) c_channel = channel.MogamiChanneltoData(ip) ans = c_channel.delfile_req(files) c_channel.close_req() c_channel.finalize()
def rename(self, oldpath, newpath): MogamiLog.debug(oldpath + ' -> ' + newpath) try: self.meta_rep.rename(oldpath, newpath) ans = 0 except os.error, e: ans = e.errno
def rmdir(self, path): MogamiLog.debug("path=%s" % (path)) try: self.meta_rep.rmdir(path) ans = 0 except os.error, e: ans = e.errno
def dataadd(self, rootpath): ip = self.c_channel.getpeername() self.sysinfo.add_data_server(ip, rootpath) MogamiLog.info("delete data server IP: %s" % ip) MogamiLog.info("Now there are %d data servers." % len(self.sysinfo.data_list))
def mkdir(self, path, mode): MogamiLog.debug("path = %s mode = %o" % (path, mode)) try: self.meta_rep.mkdir(path, mode) ans = 0 except os.error, e: ans = e.errno
def __init__(self, metaaddr, rootpath): """This is the function of MogamiMeta's init. @param metaaddr ip address or hostname of metadata server @param rootpath path of directory to store data into """ # basic information self.metaaddr = metaaddr self.rootpath = os.path.abspath(rootpath) self.filedict = MogamiLocalFileDict() # check directory for data files assert os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK) # Initialization of Log. MogamiLog.init(MogamiLog.TYPE_DATA, conf.data_loglevel) MogamiLog.info("Start initialization...") MogamiLog.debug("rootpath = " + self.rootpath) # At first, connect to metadata server and send request to attend. self.m_channel = channel.MogamiChanneltoMeta() self.m_channel.connect(self.metaaddr) MogamiLog.debug("Success in creating connection to metadata server") self.m_channel.dataadd_req(self.rootpath) MogamiLog.info("Init complete!!")
def chmod(self, path, mode): MogamiLog.debug("path = %s w/ mode %s" % (path, oct(mode))) try: self.meta_rep.chmod(path, mode) ans = 0 except os.error, e: ans = e.errno
def chown(self, path, uid, gid): MogamiLog.debug("path=%s uid=%d gid=%d" % (path, uid, gid)) try: self.meta_rep.chown(path, uid, gid) ans = 0 except os.error, e: ans = e.errno
def release(self, flags): if conf.ap is True: start_t = time.time() MogamiLog.debug("** release **") MogamiLog.critical("** file log ** file:%s\tcmd:%s\tlocal:%s\tread:%s\twrite:%s" % ( self.path, self.cmd, str(self.local), str(self.read_size), str(self.write_size))) fsize = self.mogami_file.release(flags) ans = m_channel.release_req(self.path, fsize) # delete file size cache if self.path in file_size_dict: del file_size_dict[self.path] if conf.ap is True: # prepare data to tell access pattern myname = m_channel.getmyname() (read_data, write_data) = self.access_pattern.mk_form_data() pid = self.access_pattern.pid cmd_args = self.access_pattern.cmd_args path = self.access_pattern.path # get pids of parents parents_list = self.access_pattern.parents_list end_t = time.time() self.took_time += end_t - start_t # put file access history to repository if cmd_args != None: file_access_rep.put_ap((cmd_args, pid, path, myname, self.took_time, self.created, read_data, write_data), pid, parents_list) return 0
def read(self, length, offset): """read handler. @param length request size of read @param offset offset of read request @return data read from file (may return errno with error) """ if conf.ap is True: start_t = time.time() MogamiLog.debug("**read offset=%d, length=%d" % (offset, length)) ret_buf = self.mogami_file.read(length, offset) if conf.ap is True: end_t = time.time() self.access_pattern.insert_data(self.access_pattern.read, offset, len(ret_buf)) self.took_time += end_t - start_t with self.ap_lock: self.read_size += len(ret_buf) return ret_buf
def symlink(self, frompath, topath): MogamiLog.debug("frompath = %s, topath = %s" % (frompath, topath)) try: self.meta_rep.symlink(frompath, topath) ans = 0 except os.error, e: ans = e.errno
def datadel(self, ): ip = self.c_channel.getpeername() ret = self.sysinfo.remove_data_server(ip) if ret == True: MogamiLog.info("delete data server IP: %s" % ip) MogamiLog.info("Now there are %d data servers." % len(self.sysinfo.data_list))
def __init__(self, sysinfo, meta_rep): MogamiLog.debug("== start daemon on metadata server ==") daemons.MogamiDaemons.__init__(self) self.meta_rep = meta_rep self.delfile_q = sysinfo.delfile_q self.repfile_q = sysinfo.repfile_q self.sock_list =[] self.sock_dict = {}
def readlink(self, path): MogamiLog.debug("path = %s" % path) try: result = self.meta_rep.readlink(path) ans = 0 except os.error, e: ans = e.errno result = None
def getattr(self, path): MogamiLog.debug("path = %s" % path) try: st_dict = self.meta_rep.getattr(path) ans = 0 except os.error, e: MogamiLog.debug("stat error!") ans = e.errno st_dict = None
def unlink(self, path, async): MogamiLog.debug("path = %s" % path) try: (dest, dest_path) = self.meta_rep.unlink(path) if async is True: ans = 0 if dest != None: self.sysinfo.add_delfile(dest, dest_path) else:
def access(self, path, mode): MogamiLog.debug("path = %s" % (path)) try: if self.meta_rep.access(path, mode) == True: ans = 0 else: ans = errno.EACCES except os.error, e: ans = e.errno
def register_ramfiles(self, add_file_list): """register files in list to files to manage on memory @param add_file_list """ ramfile_list.extend(add_file_list) MogamiLog.debug("** register ramfiles **") MogamiLog.debug("add files = " + str(add_file_list))
def run(self, ): while True: daemons_alive = threading.enumerate() for d in self.daemons: if d not in daemons_alive: d.join() MogamiLog.debug("** join thread **") self.daemons.remove(d) time.sleep(self.sleep_time)
def release(self, path, fsize): """release handler. @param fd file discripter @param writelen size of data to be written """ try: self.meta_rep.release(path, fsize) except Exception, e: MogamiLog.error("OSError in release (%s)" % (e))
def truncate(self, path, length): MogamiLog.debug("path = %s. length = %d" % (path, length)) try: f = open(path, "r+") f.truncate(length) f.close() ans = 0 except Exception, e: MogamiLog.error("have an error (%s)" % (e)) ans = e.errno
def access(self, path, mode): """access handler. @param path path to access @param mode mode to access @return 0 on success, errno on error """ MogamiLog.debug("** access **" + path + str(mode)) ans = m_channel.access_req(path, mode) return -ans
def mkdir(self, path, mode): """mkdir handler. @param path directory path to mkdir @param mode permission of the directory to create @return 0 on success, errno on error """ MogamiLog.debug("** mkdir **" + path + str(mode)) ans = m_channel.mkdir_req(path, mode) return -ans
def chown(self, path, uid, gid): """chown handler. @param path path to change owner of @param uid user id of new owner @param gid group id of new owner """ MogamiLog.debug('** chown ** ' + path + str(uid) + str(gid)) ans = m_channel.chown_req(path, uid, gid) return -ans
def symlink(self, frompath, topath): """symlink handler. @param frompath @param topath """ MogamiLog.debug("** symlink ** frompath = %s, topath = %s" % (frompath, topath)) ans = m_channel.symlink_req(frompath, topath) return -ans
def sendall(self, data): """send all data. This function may return errno, when send error occurs. @param data data to send """ try: self.sock.sendall(data) except Exception, e: MogamiLog.debug("** Error in sending data **") return None
def send_replication_request(self, path, org, org_path, dest, dest_path): MogamiLog.debug("file replication request was sent (%s: %s -> %s: %s)" % (org, org_path, dest, dest_path)) c_channel = channel.MogamiChanneltoData(org) c_channel.filerep_req(path, org_path, dest, dest_path) sock_id = c_channel.sock.fileno() self.sock_list.append(sock_id) self.sock_dict[sock_id] = (c_channel, path, org, org_path, dest, dest_path)