def open(self, path, flag, mode):
"""open handler.
@param path file path
@param flag flags for open(2)
@param mode open mode (may be empty tuple): actual value is mode[0]
"""
if self.meta_rep.access(path, os.F_OK) == True:
# When the required file exist...
try:
MogamiLog.debug("!!find the file %s w/ %o" % (path, flag))
affinity = self.c_channel.getpeername()
(dest, data_path, fsize) = self.meta_rep.open(
path, flag, mode, affinity)
# create data to send
ans = 0
created = False
except Exception, e:
MogamiLog.debug("!!find the file but error for %s (%s)" %
(path, e))
try:
ans = e.errno
except Exception, e:
ans = errno.ENOENT
dest = None
fd = None
data_path = None
fsize = None
created = False
def utime(self, path, times):
MogamiLog.debug("path = %s, times = %s" % (path, str(times)))
try:
self.meta_rep.utime(path, times)
ans = 0
except os.error, e:
ans = e.errno
def fgetattr(self, fd):
try:
st = os.fstat(fd)
senddata = [0, st]
except os.error, e:
MogamiLog.debug("OSError in fgetattr (%s)" % (e))
senddata = [e.errno, 'null']
def send_delete_request(self, ip, files):
MogamiLog.debug("file delete request was sent (%s -> %s)" %
(ip, str(files)))
c_channel = channel.MogamiChanneltoData(ip)
ans = c_channel.delfile_req(files)
c_channel.close_req()
c_channel.finalize()
def rename(self, oldpath, newpath):
MogamiLog.debug(oldpath + ' -> ' + newpath)
try:
self.meta_rep.rename(oldpath, newpath)
ans = 0
except os.error, e:
ans = e.errno
def rmdir(self, path):
MogamiLog.debug("path=%s" % (path))
try:
self.meta_rep.rmdir(path)
ans = 0
except os.error, e:
ans = e.errno
def dataadd(self, rootpath):
ip = self.c_channel.getpeername()
self.sysinfo.add_data_server(ip, rootpath)
MogamiLog.info("delete data server IP: %s" % ip)
MogamiLog.info("Now there are %d data servers." %
len(self.sysinfo.data_list))
def mkdir(self, path, mode):
MogamiLog.debug("path = %s mode = %o" % (path, mode))
try:
self.meta_rep.mkdir(path, mode)
ans = 0
except os.error, e:
ans = e.errno
def __init__(self, metaaddr, rootpath):
"""This is the function of MogamiMeta's init.
@param metaaddr ip address or hostname of metadata server
@param rootpath path of directory to store data into
"""
# basic information
self.metaaddr = metaaddr
self.rootpath = os.path.abspath(rootpath)
self.filedict = MogamiLocalFileDict()
# check directory for data files
assert os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK)
# Initialization of Log.
MogamiLog.init(MogamiLog.TYPE_DATA, conf.data_loglevel)
MogamiLog.info("Start initialization...")
MogamiLog.debug("rootpath = " + self.rootpath)
# At first, connect to metadata server and send request to attend.
self.m_channel = channel.MogamiChanneltoMeta()
self.m_channel.connect(self.metaaddr)
MogamiLog.debug("Success in creating connection to metadata server")
self.m_channel.dataadd_req(self.rootpath)
MogamiLog.info("Init complete!!")
def chmod(self, path, mode):
MogamiLog.debug("path = %s w/ mode %s" % (path, oct(mode)))
try:
self.meta_rep.chmod(path, mode)
ans = 0
except os.error, e:
ans = e.errno
def chown(self, path, uid, gid):
MogamiLog.debug("path=%s uid=%d gid=%d" % (path, uid, gid))
try:
self.meta_rep.chown(path, uid, gid)
ans = 0
except os.error, e:
ans = e.errno
def release(self, flags):
if conf.ap is True:
start_t = time.time()
MogamiLog.debug("** release **")
MogamiLog.critical("** file log ** file:%s\tcmd:%s\tlocal:%s\tread:%s\twrite:%s" % (
self.path, self.cmd, str(self.local), str(self.read_size), str(self.write_size)))
fsize = self.mogami_file.release(flags)
ans = m_channel.release_req(self.path, fsize)
# delete file size cache
if self.path in file_size_dict:
del file_size_dict[self.path]
if conf.ap is True:
# prepare data to tell access pattern
myname = m_channel.getmyname()
(read_data, write_data) = self.access_pattern.mk_form_data()
pid = self.access_pattern.pid
cmd_args = self.access_pattern.cmd_args
path = self.access_pattern.path
# get pids of parents
parents_list = self.access_pattern.parents_list
end_t = time.time()
self.took_time += end_t - start_t
# put file access history to repository
if cmd_args != None:
file_access_rep.put_ap((cmd_args, pid, path, myname,
self.took_time, self.created,
read_data, write_data),
pid, parents_list)
return 0
def read(self, length, offset):
"""read handler.
@param length request size of read
@param offset offset of read request
@return data read from file (may return errno with error)
"""
if conf.ap is True:
start_t = time.time()
MogamiLog.debug("**read offset=%d, length=%d" % (offset, length))
ret_buf = self.mogami_file.read(length, offset)
if conf.ap is True:
end_t = time.time()
self.access_pattern.insert_data(self.access_pattern.read, offset,
len(ret_buf))
self.took_time += end_t - start_t
with self.ap_lock:
self.read_size += len(ret_buf)
return ret_buf
def symlink(self, frompath, topath):
MogamiLog.debug("frompath = %s, topath = %s" % (frompath, topath))
try:
self.meta_rep.symlink(frompath, topath)
ans = 0
except os.error, e:
ans = e.errno
def datadel(self, ):
ip = self.c_channel.getpeername()
ret = self.sysinfo.remove_data_server(ip)
if ret == True:
MogamiLog.info("delete data server IP: %s" % ip)
MogamiLog.info("Now there are %d data servers." %
len(self.sysinfo.data_list))
def __init__(self, sysinfo, meta_rep):
MogamiLog.debug("== start daemon on metadata server ==")
daemons.MogamiDaemons.__init__(self)
self.meta_rep = meta_rep
self.delfile_q = sysinfo.delfile_q
self.repfile_q = sysinfo.repfile_q
self.sock_list =[]
self.sock_dict = {}
def readlink(self, path):
MogamiLog.debug("path = %s" % path)
try:
result = self.meta_rep.readlink(path)
ans = 0
except os.error, e:
ans = e.errno
result = None
def getattr(self, path):
MogamiLog.debug("path = %s" % path)
try:
st_dict = self.meta_rep.getattr(path)
ans = 0
except os.error, e:
MogamiLog.debug("stat error!")
ans = e.errno
st_dict = None
def unlink(self, path, async):
MogamiLog.debug("path = %s" % path)
try:
(dest, dest_path) = self.meta_rep.unlink(path)
if async is True:
ans = 0
if dest != None:
self.sysinfo.add_delfile(dest, dest_path)
else:
def access(self, path, mode):
MogamiLog.debug("path = %s" % (path))
try:
if self.meta_rep.access(path, mode) == True:
ans = 0
else:
ans = errno.EACCES
except os.error, e:
ans = e.errno
def register_ramfiles(self, add_file_list):
"""register files in list to files to manage on memory
@param add_file_list
"""
ramfile_list.extend(add_file_list)
MogamiLog.debug("** register ramfiles **")
MogamiLog.debug("add files = " + str(add_file_list))
def run(self, ):
while True:
daemons_alive = threading.enumerate()
for d in self.daemons:
if d not in daemons_alive:
d.join()
MogamiLog.debug("** join thread **")
self.daemons.remove(d)
time.sleep(self.sleep_time)
def release(self, path, fsize):
"""release handler.
@param fd file discripter
@param writelen size of data to be written
"""
try:
self.meta_rep.release(path, fsize)
except Exception, e:
MogamiLog.error("OSError in release (%s)" % (e))
def truncate(self, path, length):
MogamiLog.debug("path = %s. length = %d" % (path, length))
try:
f = open(path, "r+")
f.truncate(length)
f.close()
ans = 0
except Exception, e:
MogamiLog.error("have an error (%s)" % (e))
ans = e.errno
def access(self, path, mode):
"""access handler.
@param path path to access
@param mode mode to access
@return 0 on success, errno on error
"""
MogamiLog.debug("** access **" + path + str(mode))
ans = m_channel.access_req(path, mode)
return -ans
def mkdir(self, path, mode):
"""mkdir handler.
@param path directory path to mkdir
@param mode permission of the directory to create
@return 0 on success, errno on error
"""
MogamiLog.debug("** mkdir **" + path + str(mode))
ans = m_channel.mkdir_req(path, mode)
return -ans
def chown(self, path, uid, gid):
"""chown handler.
@param path path to change owner of
@param uid user id of new owner
@param gid group id of new owner
"""
MogamiLog.debug('** chown ** ' + path + str(uid) + str(gid))
ans = m_channel.chown_req(path, uid, gid)
return -ans
def symlink(self, frompath, topath):
"""symlink handler.
@param frompath
@param topath
"""
MogamiLog.debug("** symlink ** frompath = %s, topath = %s" %
(frompath, topath))
ans = m_channel.symlink_req(frompath, topath)
return -ans
def sendall(self, data):
"""send all data.
This function may return errno, when send error occurs.
@param data data to send
"""
try:
self.sock.sendall(data)
except Exception, e:
MogamiLog.debug("** Error in sending data **")
return None
def send_replication_request(self, path, org, org_path, dest, dest_path):
MogamiLog.debug("file replication request was sent (%s: %s -> %s: %s)" %
(org, org_path, dest, dest_path))
c_channel = channel.MogamiChanneltoData(org)
c_channel.filerep_req(path, org_path, dest, dest_path)
sock_id = c_channel.sock.fileno()
self.sock_list.append(sock_id)
self.sock_dict[sock_id] = (c_channel, path, org, org_path,
dest, dest_path)