def chown(self, path, uid, gid):
MogamiLog.debug("path=%s uid=%d gid=%d" % (path, uid, gid))
try:
os.chown(path, uid, gid)
ans = 0
except os.error, e:
ans = e.errno
def utime(self, path, times):
MogamiLog.debug("path = %s, times = %s" % (path, str(times)))
try:
os.utime(path, times)
ans = 0
except os.error, e:
ans = e.errno
def rmdir(self, path):
MogamiLog.debug("path=%s" % (path))
try:
os.rmdir(path)
ans = 0
except os.error, e:
ans = e.errno
def __init__(self, fsize, dest, data_path, flag, *mode):
MogamiFile.__init__(self, fsize)
self.remote = True
self.dest = dest
self.data_path = data_path
self.flag = flag
self.mode = mode
self.prenum = 1
# calculation of the number of blocks
self.blnum = self.fsize / conf.blsize
if self.fsize % conf.blsize != 0:
self.blnum += 1
# initialization of read buffer
self.r_data = tuple([MogamiBlock() for i in range(self.blnum + 1)])
self.r_buflock = threading.Lock()
MogamiLog.debug("create r_data 0-%d block" % (len(self.r_data)-1))
# initialization of write buffer
self.w_list = []
self.w_data = cStringIO.StringIO()
self.w_buflock = threading.Lock()
self.w_len = 0
# for buffer of dirty data
self.dirty_dict = {}
def chmod(self, path, mode):
MogamiLog.debug("path = %s w/ mode %s" % (path, oct(mode)))
try:
os.chmod(path, mode)
ans = 0
except os.error, e:
ans = e.errno
def release(self, flags):
if conf.ap == True:
start_t = time.time()
MogamiLog.debug("** release **")
fsize = self.mogami_file.release(flags)
ans = m_channel.release_req(self.metafd, fsize)
# delete file size cache
if self.path in file_size_dict:
del file_size_dict[self.path]
if conf.ap == True:
# prepare data to tell access pattern
myname = m_channel.getmyname()
(read_data, write_data) = self.access_pattern.mk_form_data()
pid = self.access_pattern.pid
cmd_args = self.access_pattern.cmd_args
path = self.access_pattern.path
end_t = time.time()
self.took_time += end_t - start_t
if cmd_args != None:
# process access pattern history (may be inserted to queue)
file_access_queue.put((cmd_args, pid, path, myname,
self.took_time, self.created,
read_data, write_data))
return 0
def rename(self, oldpath, newpath):
MogamiLog.debug(oldpath + ' -> ' + newpath)
try:
os.rename(oldpath, newpath)
ans = 0
except os.error, e:
ans = e.errno
def rename(self, oldpath, newpath):
MogamiLog.debug(oldpath + ' -> ' + newpath)
try:
os.rename(oldpath, newpath)
ans = 0
except os.error, e:
ans = e.errno
def release(self, flags):
if conf.ap == True:
start_t = time.time()
MogamiLog.debug("** release **")
fsize = self.mogami_file.release(flags)
ans = m_channel.release_req(self.metafd, fsize)
# delete file size cache
if self.path in file_size_dict:
del file_size_dict[self.path]
if conf.ap == True:
# prepare data to tell access pattern
myname = m_channel.getmyname()
(read_data, write_data) = self.access_pattern.mk_form_data()
pid = self.access_pattern.pid
cmd_args = self.access_pattern.cmd_args
path = self.access_pattern.path
end_t = time.time()
self.took_time += end_t - start_t
if cmd_args != None:
# process access pattern history (may be inserted to queue)
file_access_queue.put(
(cmd_args, pid, path, myname, self.took_time,
self.created, read_data, write_data))
return 0
def chmod(self, path, mode):
MogamiLog.debug("path = %s w/ mode %s" % (path, oct(mode)))
try:
os.chmod(path, mode)
ans = 0
except os.error, e:
ans = e.errno
def __init__(self, metaaddr, rootpath, mogami_dir):
"""This is the function of MogamiMeta's init.
@param metaaddr ip address or hostname of metadata server
@param rootpath path of directory to store data into
@param mogami_dir path of mogami's root directory
"""
# basic information of metadata server
self.metaaddr = metaaddr
self.rootpath = os.path.abspath(rootpath)
self.mogami_dir = mogami_dir
# check directory for data files
assert os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK)
# Initialization of Log.
MogamiLog.init("data", conf.data_loglevel)
MogamiLog.info("Start initialization...")
MogamiLog.debug("rootpath = " + self.rootpath)
# At first, connect to metadata server and send request to attend.
self.m_channel = Channel.MogamiChanneltoMeta()
self.m_channel.connect(self.metaaddr)
MogamiLog.debug("Success in creating connection to metadata server")
self.m_channel.dataadd_req(self.rootpath)
MogamiLog.debug("Init complete!!")
def symlink(self, frompath, topath):
MogamiLog.debug("frompath = %s, topath = %s" % (frompath, topath))
try:
os.symlink(frompath, topath)
ans = 0
except os.error, e:
ans = e.errno
def chown(self, path, uid, gid):
MogamiLog.debug("path=%s uid=%d gid=%d" % (path, uid, gid))
try:
os.chown(path, uid, gid)
ans = 0
except os.error, e:
ans = e.errno
def mkdir(self, path, mode):
MogamiLog.debug("path = %s mode = %o" % (path, mode))
try:
os.mkdir(path, mode)
ans = 0
except os.error, e:
ans = e.errno
def __init__(self, metaaddr, rootpath, mogami_dir):
"""This is the function of MogamiMeta's init.
@param metaaddr ip address or hostname of metadata server
@param rootpath path of directory to store data into
@param mogami_dir path of mogami's root directory
"""
# basic information of metadata server
self.metaaddr = metaaddr
self.rootpath = os.path.abspath(rootpath)
self.mogami_dir = mogami_dir
# check directory for data files
assert os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK)
# Initialization of Log.
MogamiLog.init("data", conf.data_loglevel)
MogamiLog.info("Start initialization...")
MogamiLog.debug("rootpath = " + self.rootpath)
# At first, connect to metadata server and send request to attend.
self.m_channel = Channel.MogamiChanneltoMeta()
self.m_channel.connect(self.metaaddr)
MogamiLog.debug("Success in creating connection to metadata server")
self.m_channel.dataadd_req(self.rootpath)
MogamiLog.debug("Init complete!!")
def __init__(self, fsize, dest, data_path, flag, *mode):
MogamiFile.__init__(self, fsize)
self.remote = True
self.dest = dest
self.data_path = data_path
self.flag = flag
self.mode = mode
self.prenum = 1
# calculation of the number of blocks
self.blnum = self.fsize / conf.blsize
if self.fsize % conf.blsize != 0:
self.blnum += 1
# initialization of read buffer
self.r_data = tuple([MogamiBlock() for i in range(self.blnum + 1)])
self.r_buflock = threading.Lock()
MogamiLog.debug("create r_data 0-%d block" % (len(self.r_data) - 1))
# initialization of write buffer
self.w_list = []
self.w_data = cStringIO.StringIO()
self.w_buflock = threading.Lock()
self.w_len = 0
# for buffer of dirty data
self.dirty_dict = {}
def mkdir(self, path, mode):
MogamiLog.debug("path = %s mode = %o" % (path, mode))
try:
os.mkdir(path, mode)
ans = 0
except os.error, e:
ans = e.errno
def symlink(self, frompath, topath):
MogamiLog.debug("frompath = %s, topath = %s" % (frompath, topath))
try:
os.symlink(frompath, topath)
ans = 0
except os.error, e:
ans = e.errno
def rmdir(self, path):
MogamiLog.debug("path=%s" % (path))
try:
os.rmdir(path)
ans = 0
except os.error, e:
ans = e.errno
def read(self, length, offset):
"""read handler.
return strings read from file
@param length request size of read
@param offset offset of read request
"""
MogamiLog.debug("**read offset=%d, length=%d" % (offset, length))
# check access pattern
#self.access_pattern.check_mode(offset, length)
#self.access_pattern.change_info(offset, ret_str.tell(),
# last_readbl)
"""
if conf.prefetch == True:
if last_readbl != self.access_pattern.last_bl:
#MogamiLog.debug("read: prenum %d \n" % (self.prenum))
if conf.force_prenum == True:
self.prenum = conf.prenum
blnum_list = []
prereq_list = self.access_pattern.return_need_block(
(last_readbl + 1) * 1024 * 1024, self.prenum,
self.blnum)
for prereq in prereq_list:
if prereq > self.blnum:
break
if self.bldata[prereq].state == 0:
blnum_list.append(prereq)
if len(blnum_list) != 0:
self.request_prefetch(blnum_list)
self.calc_time += end_t - start_t"""
return self.mogami_file.read(length, offset)
def __init__(self, path, flag, *mode):
"""Initializer called when opened.
@param path file path
@param flag flags with open(2)
@param *mode file open mode (may not be specified)
"""
MogamiLog.debug("** open ** path = %s, flag = %s, mode = %s" %
(path, str(flag), str(mode)))
if conf.ap == True:
start_t = time.time()
# parse argurments
self.path = path
self.flag = flag
self.mode = mode
(ans, dest, self.metafd, data_path, self.fsize,
self.created) = m_channel.open_req(path, flag, *mode)
if ans != 0: # error on metadata server
e = IOError()
e.errno = ans
raise e
if dest == 'self':
self.mogami_file = FileManager.MogamiLocalFile(
self.fsize, data_path, flag, *mode)
else:
self.mogami_file = FileManager.MogamiRemoteFile(
self.fsize, dest, data_path, flag, *mode)
ans = self.mogami_file.create_connections(channels)
if ans != 0:
MogamiLog.error("open error !!")
e = IOError()
e.errno = ans
raise e
# register file size to file size dictionary
file_size_dict[path] = self.fsize
if conf.ap == True:
"""Get Id list to know pid.
list = {gid: pid: uid}
And then get the command from pid.
"""
try:
id_list = self.GetContext()
pid = id_list['pid']
f = open(os.path.join("/proc", str(pid), "cmdline"), 'r')
cmd_args = f.read().rsplit('\x00')[:-1]
except Exception, e:
# with any error, pass this part of process
cmd_args = None
pid = -1
self.access_pattern = FileManager.MogamiAccessPattern(
path, cmd_args, pid)
end_t = time.time()
self.took_time = end_t - start_t
def __init__(self, path, flag, *mode):
"""Initializer called when opened.
@param path file path
@param flag flags with open(2)
@param *mode file open mode (may not be specified)
"""
MogamiLog.debug("** open ** path = %s, flag = %s, mode = %s" %
(path, str(flag), str(mode)))
if conf.ap == True:
start_t = time.time()
# parse argurments
self.path = path
self.flag = flag
self.mode = mode
(ans, dest, self.metafd, data_path, self.fsize,
self.created) = m_channel.open_req(path, flag, *mode)
if ans != 0: # error on metadata server
e = IOError()
e.errno = ans
raise e
if dest == 'self':
self.mogami_file = FileManager.MogamiLocalFile(
self.fsize, data_path, flag, *mode)
else:
self.mogami_file = FileManager.MogamiRemoteFile(
self.fsize, dest, data_path, flag, *mode)
ans = self.mogami_file.create_connections(channels)
if ans != 0:
MogamiLog.error("open error !!")
e = IOError()
e.errno = ans
raise e
# register file size to file size dictionary
file_size_dict[path] = self.fsize
if conf.ap == True:
"""Get Id list to know pid.
list = {gid: pid: uid}
And then get the command from pid.
"""
try:
id_list = self.GetContext()
pid = id_list['pid']
f = open(os.path.join("/proc", str(pid), "cmdline"), 'r')
cmd_args = f.read().rsplit('\x00')[:-1]
except Exception, e:
# with any error, pass this part of process
cmd_args = None
pid = -1
self.access_pattern = FileManager.MogamiAccessPattern(
path, cmd_args, pid)
end_t = time.time()
self.took_time = end_t - start_t
def unlink(self, path):
"""unlink handler.
@param path path name to unlink
"""
MogamiLog.debug("** unlink ** path = %s" % (path, ))
ans = m_channel.unlink_req(path)
return -ans
def unlink(self, path):
"""unlink handler.
@param path path name to unlink
"""
MogamiLog.debug("** unlink ** path = %s" % (path, ))
ans = m_channel.unlink_req(path)
return -ans
def data_add(self, ip, rootpath):
self.sysinfo.add_data_server(ip, rootpath)
print "add data server IP:", ip
print "Now %d data servers are." % len(self.sysinfo.data_list)
MogamiLog.info("delete data server IP: %s" % ip)
MogamiLog.info("Now there are %d data servers." %
len(self.sysinfo.data_list))
def unlink(self, path):
MogamiLog.debug("path = %s" % path)
if os.path.isfile(path):
try:
(dest, data_path, fsize) = meta_file_info(path)
self.sysinfo.add_delfile(dest, data_path)
except Exception, e:
MogamiLog.error("cannot remove file contents of %s", path)
def utime(self, path, times):
print "** utime **"
MogamiLog.debug("path = %s, times = %s" % (path, str(times)))
try:
os.utime(path, times)
ans = 0
except os.error, e:
ans = e.errno
def data_add(self, ip, rootpath):
self.sysinfo.add_data_server(ip, rootpath)
print "add data server IP:", ip
print "Now %d data servers are." % len(self.sysinfo.data_list)
MogamiLog.info("delete data server IP: %s" % ip)
MogamiLog.info("Now there are %d data servers." %
len(self.sysinfo.data_list))
def unlink(self, path):
MogamiLog.debug("path = %s" % path)
if os.path.isfile(path):
try:
(dest, data_path, fsize) = meta_file_info(path)
self.sysinfo.add_delfile(dest, data_path)
except Exception, e:
MogamiLog.error("cannot remove file contents of %s", path)
def data_del(self, ip):
ret = self.sysinfo.remove_data_server(ip)
if ret == True:
print "delete data server IP:", ip
print "Now %d data servers are." % len(self.sysinfo.data_list)
MogamiLog.info("delete data server IP: %s" % ip)
MogamiLog.info("Now there are %d data servers." %
len(self.sysinfo.data_list))
def register_ramfiles(self, add_file_list):
"""register files in list to files to manage on memory
@param add_file_list
"""
ramfile_list.extend(add_file_list)
MogamiLog.debug("** register ramfiles **")
MogamiLog.debug("add files = " + str(add_file_list))
def register_ramfiles(self, add_file_list):
"""register files in list to files to manage on memory
@param add_file_list
"""
ramfile_list.extend(add_file_list)
MogamiLog.debug("** register ramfiles **")
MogamiLog.debug("add files = " + str(add_file_list))
def access(self, path, mode):
MogamiLog.debug("path = %s" % (path))
try:
if os.access(path, mode) == True:
ans = 0
else:
ans = errno.EACCES
except os.error, e:
ans = e.errno
def data_del(self, ip):
ret = self.sysinfo.remove_data_server(ip)
if ret == True:
print "delete data server IP:", ip
print "Now %d data servers are." % len(self.sysinfo.data_list)
MogamiLog.info("delete data server IP: %s" % ip)
MogamiLog.info("Now there are %d data servers." %
len(self.sysinfo.data_list))
def readlink(self, path):
print "path = " + path
MogamiLog.debug("path = %s" % path)
try:
result = os.readlink(path)
ans = 0
except os.error, e:
ans = e.errno
result = None
def access(self, path, mode):
MogamiLog.debug("path = %s" % (path))
try:
if os.access(path, mode) == True:
ans = 0
else:
ans = errno.EACCES
except os.error, e:
ans = e.errno
def readlink(self, path):
print "path = " + path
MogamiLog.debug("path = %s" % path)
try:
result = os.readlink(path)
ans = 0
except os.error, e:
ans = e.errno
result = None
def access(self, path, mode):
"""access handler.
@param path path to access
@param mode mode to access
@return 0 on success, errno on error
"""
MogamiLog.debug("** access **" + path + str(mode))
ans = m_channel.access_req(path, mode)
return -ans
def chown(self, path, uid, gid):
"""chown handler.
@param path path to change owner of
@param uid user id of new owner
@param gid group id of new owner
"""
MogamiLog.debug('** chown ** ' + path + str(uid) + str(gid))
ans = m_channel.chown_req(path, uid, gid)
return -ans
def truncate(self, path, length):
MogamiLog.debug("path = %s. length = %d" % (path, length))
try:
f = open(path, "r+")
f.truncate(length)
f.close()
ans = 0
except Exception, e:
MogamiLog.error("have an error (%s)" % (e))
ans = e.errno
def truncate(self, path, length):
MogamiLog.debug("path = %s. length = %d" % (path, length))
try:
f = open(path, "r+")
f.truncate(length)
f.close()
ans = 0
except Exception, e:
MogamiLog.error("have an error (%s)" % (e))
ans = e.errno
def access(self, path, mode):
"""access handler.
@param path path to access
@param mode mode to access
@return 0 on success, errno on error
"""
MogamiLog.debug("** access **" + path + str(mode))
ans = m_channel.access_req(path, mode)
return -ans
def mkdir(self, path, mode):
"""mkdir handler.
@param path directory path to mkdir
@param mode permission of the directory to create
@return 0 on success, errno on error
"""
MogamiLog.debug("** mkdir **" + path + str(mode))
ans = m_channel.mkdir_req(path, mode)
return -ans
def release(self, flags):
MogamiLog.debug("** release **")
fsize = self.mogami_file.release(flags)
ans = m_channel.release_req(self.metafd, fsize)
# delete file size cache
if self.path in file_size_dict:
del file_size_dict[self.path]
return 0
def symlink(self, frompath, topath):
"""symlink handler.
@param frompath
@param topath
"""
MogamiLog.debug("** symlink ** frompath = %s, topath = %s" %
(frompath, topath))
ans = m_channel.symlink_req(frompath, topath)
return -ans
def symlink(self, frompath, topath):
"""symlink handler.
@param frompath
@param topath
"""
MogamiLog.debug("** symlink ** frompath = %s, topath = %s" %
(frompath, topath))
ans = m_channel.symlink_req(frompath, topath)
return -ans
def mkdir(self, path, mode):
"""mkdir handler.
@param path directory path to mkdir
@param mode permission of the directory to create
@return 0 on success, errno on error
"""
MogamiLog.debug("** mkdir **" + path + str(mode))
ans = m_channel.mkdir_req(path, mode)
return -ans
def chmod(self, path, mode):
"""chmod handler.
@param path path to change permission of
@param mode permission to change
"""
MogamiLog.debug("** chmod ** path = %s, mode = %s" % (path, oct(mode)))
ans = m_channel.chmod_req(path, mode)
if ans != 0:
return -ans
def chown(self, path, uid, gid):
"""chown handler.
@param path path to change owner of
@param uid user id of new owner
@param gid group id of new owner
"""
MogamiLog.debug('** chown ** ' + path + str(uid) + str(gid))
ans = m_channel.chown_req(path, uid, gid)
return -ans
def open(self, path, flag, *mode):
start_t = time.time()
MogamiLog.debug("path = %s, flag = %s, mode = %s" %
(path, str(flag), str(mode)))
flag = flag & ~os.O_EXCL
try:
fd = os.open(path, flag, *mode)
ans = 0
except Exception, e:
fd = None
ans = e.errno
def rename(self, oldpath, newpath):
"""rename handler.
@param oldpath original path name before rename
@param newpath new path name after rename
"""
MogamiLog.debug("** rename ** oldpath = %s, newpath = %s" %
(oldpath, newpath))
ans = m_channel.rename_req(oldpath, newpath)
if ans != 0:
return -ans
def readdir(self, path, offset):
MogamiLog.debug("** readdir ** path = %s, offset = %s" %
(path, str(offset)))
(ans, contents) = m_channel.readdir_req(path, offset)
l = ['.', '..']
if ans == 0:
l.extend(contents)
return [fuse.Direntry(ent) for ent in l]
else:
return -ans
def readdir(self, path, offset):
MogamiLog.debug("** readdir ** path = %s, offset = %s" %
(path, str(offset)))
(ans, contents) = m_channel.readdir_req(path, offset)
l = ['.', '..']
if ans == 0:
l.extend(contents)
return [fuse.Direntry(ent) for ent in l]
else:
return -ans
def chmod(self, path, mode):
"""chmod handler.
@param path path to change permission of
@param mode permission to change
"""
MogamiLog.debug("** chmod ** path = %s, mode = %s" %
(path, oct(mode)))
ans = m_channel.chmod_req(path, mode)
if ans != 0:
return -ans
def rename(self, oldpath, newpath):
"""rename handler.
@param oldpath original path name before rename
@param newpath new path name after rename
"""
MogamiLog.debug("** rename ** oldpath = %s, newpath = %s" %
(oldpath, newpath))
ans = m_channel.rename_req(oldpath, newpath)
if ans != 0:
return -ans
def open(self, path, flag, *mode):
start_t = time.time()
MogamiLog.debug("path = %s, flag = %s, mode = %s" %
(path, str(flag), str(mode)))
flag = flag & ~os.O_EXCL
try:
fd = os.open(path, flag, *mode)
ans = 0
except Exception, e:
fd = None
ans = e.errno
def getattr(self, path):
MogamiLog.debug("path = %s" % path)
# get result of stat (ans and st)
try:
st = os.lstat(path)
ans = 0
except os.error, e:
MogamiLog.debug("stat error!")
ans = e.errno
st = None
def getattr(self, path):
MogamiLog.debug("path = %s" % path)
# get result of stat (ans and st)
try:
st = os.lstat(path)
ans = 0
except os.error, e:
MogamiLog.debug("stat error!")
ans = e.errno
st = None
def request_prefetch(self, blnum_list):
"""send request of prefetch.
@param blnum_list the list of block numbers to prefetch
"""
MogamiLog.debug("** prefetch ** required blocks = %s" %
(str(blnum_list)))
# send request to data server
self.p_channel.prefetch_req(self.datafd, blnum_list)
with self.r_buflock:
for blnum in blnum_list:
self.r_data[blnum].state = 1
def remove_data_server(self, ip):
"""remove a data server from Mogami system
@param ip ip address of node to remove
@return 0 with success, -1 with error
"""
try:
self.data_list.remove(ip)
del self.data_rootpath[ip]
return True
except Exception, e:
MogamiLog.error("cannot find %s from data servers" % (ip))
return False
