def create_connections(self, channel_repository):
"""Create connections to data server which has file contents.
In this function, send request for open to data server.
(and calculate RTT)
"""
channels = channel_repository.get_channel(self.dest)
self.d_channel = Channel.MogamiChanneltoData(self.dest)
# create a connection for prefetching
#self.p_channel = Channel.MogamiChanneltoData(self.dest)
#channel_repository.set_channel(self.dest, self.d_channel, self.p_channel)
#else:
# set channels
# self.d_channel = channels[0]
# self.p_channel = channels[1]
# send a request to data server for open
start_t = time.time()
(ans, self.datafd,
open_t) = self.d_channel.open_req(self.data_path, self.flag,
*self.mode)
end_t = time.time()
if ans != 0: # failed...with errno
self.finalize()
return ans
# on success
self.rtt = end_t - start_t - open_t
# must be 0
return ans
def run(self, ):
"""Connected from Mogami Client.
"""
self.lsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.lsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.lsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.lsock.bind(("0.0.0.0", conf.metaport))
self.lsock.listen(10)
MogamiLog.debug("Listening at the port " + str(conf.metaport))
daemons = []
thread_collector = Daemons.MogamiThreadCollector(daemons)
thread_collector.start()
threads_count = 0
delete_files_thread = MogamiDaemononMeta(self.sysinfo)
delete_files_thread.start()
while True:
(client_sock, address) = self.lsock.accept()
MogamiLog.debug("accept connnect from %s" % (str(address[0])))
client_channel = Channel.MogamiChannelforMeta()
client_channel.set_socket(client_sock)
metad = MogamiMetaHandler(client_channel, self.sysinfo)
metad.start()
daemons.append(metad)
MogamiLog.debug("Created thread name = " + metad.getName())
def run(self, ):
# create a socket to listen and accept
self.lsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.lsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.lsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.lsock.bind(("0.0.0.0", conf.dataport))
self.lsock.listen(10)
MogamiLog.debug("Listening on the port " + str(conf.dataport))
# create a thread to collect dead daemon threads
daemons = []
collector_thread = Daemons.MogamiThreadCollector(daemons)
collector_thread.start()
threads_count = 0
while True:
# connected from client
(csock, address) = self.lsock.accept()
MogamiLog.debug("accept connnect from " + str(address[0]))
client_channel = Channel.MogamiChannelforData()
client_channel.set_socket(csock)
datad = MogamiDataHandler(client_channel, self.rootpath)
datad.name = "D%d" % (threads_count)
threads_count += 1
datad.start()
daemons.append(datad)
MogamiLog.debug("Created thread name = %s (%d-th threads)" %
(datad.getName(), threads_count))
def __init__(self, metaaddr, rootpath, mogami_dir):
"""This is the function of MogamiMeta's init.
@param metaaddr ip address or hostname of metadata server
@param rootpath path of directory to store data into
@param mogami_dir path of mogami's root directory
"""
# basic information of metadata server
self.metaaddr = metaaddr
self.rootpath = os.path.abspath(rootpath)
self.mogami_dir = mogami_dir
# check directory for data files
assert os.access(self.rootpath, os.R_OK and os.W_OK and os.X_OK)
# Initialization of Log.
MogamiLog.init("data", conf.data_loglevel)
MogamiLog.info("Start initialization...")
MogamiLog.debug("rootpath = " + self.rootpath)
# At first, connect to metadata server and send request to attend.
self.m_channel = Channel.MogamiChanneltoMeta()
self.m_channel.connect(self.metaaddr)
MogamiLog.debug("Success in creating connection to metadata server")
self.m_channel.dataadd_req(self.rootpath)
MogamiLog.debug("Init complete!!")
def __init__(self, pipepath):
Daemons.MogamiDaemons.__init__(self)
self.pipepath = pipepath
if os.access(self.pipepath, os.F_OK) == True:
os.remove(self.pipepath)
self.file_access_dict = {} # {pid: file_accesses (list) }
self.channel = Channel.MogamiChanneltoTellAP(self.pipepath)
def truncate(self, path, length):
MogamiLog.debug('** truncate ** path = %s, length = %d' %
(path, length))
(ans, dest, filename) = m_channel.truncate_req(path, length)
if ans != 0:
return -ans
c_channel = Channel.MogamiChanneltoData(dest)
ans = c_channel.truncate_req(filename, length)
c_channel.finalize()
# if truncate was succeeded, cache of file size should be changed
if ans == 0:
file_size_dict[path] = length
return -ans
def ask_file_access(pid, path):
ch = Channel.MogamiChanneltoAskAP(path)
ap_list = ch.file_access_req(pid)
return ap_list
def send_delete_request(self, ip, files):
c_channel = Channel.MogamiChanneltoData(ip)
ans = c_channel.delfile_req(files)
c_channel.close_req()
c_channel.finalize()
import Queue
import sys
import time
sys.path.append(os.pardir)
# mogami's original modules
from conf import conf
from libs import Channel
from libs import DBMng
from libs import System
from libs import Tips
from libs import Daemons
from libs import FileManager
from libs.System import MogamiLog
m_channel = Channel.MogamiChanneltoMeta()
daemons = []
file_size_dict = {}
channels = Channel.MogamiChannelRepository()
file_access_queue = Queue.Queue()
class MogamitoTellAccessPattern(Daemons.MogamiDaemons):
def __init__(self, pipepath):
Daemons.MogamiDaemons.__init__(self)
self.pipepath = pipepath
if os.access(self.pipepath, os.F_OK) == True:
os.remove(self.pipepath)
self.file_access_dict = {} # {pid: file_accesses (list) }
self.channel = Channel.MogamiChanneltoTellAP(self.pipepath)
def finalize(self, ):
if self.m_channel == None:
self.m_channel = Channel.MogamiChanneltoMeta()
self.m_channel.connect(self.metaaddr, conf.metaport)
self.m_channel.datadel_req()
self.m_channel.finalize()