def __init__(self, config, test=False):
"""@config: ConfigParser() object"""
self._config = config
ds_ip = config.get('dataserver', 'ds_ip')
ds_port = config.getint('dataserver', 'ds_port')
self._server = eventlet.listen((ds_ip, ds_port))
thread_num = config.getint('threads', 'thread_num')
self._pool = eventlet.GreenPool(thread_num)
store_dir = config.get('storage', 'chunk_store_dir')
logging.info('get store dir: %s', store_dir)
store_type = config.get('storage', 'type')
if store_type == DS_LOCAL:
self._ds = DSStore(store_dir)
elif store_type == DS_MEMORY:
self._ds = MemStore(store_dir)
else:
self._ds = DSStore(store_dir)
# register to mds when not in testmode
if not test:
mds_ip = config.get('mds', 'mds_ip')
mds_port = config.getint('mds', 'mds_port')
self._add_ds(mds_ip, mds_port)
# handler for request to ds
self._handlers = {
OP_ADD_CHUNK: self._handle_add_chunk,
OP_DELETE_CHUNK: self._handle_delete_chunk,
OP_GET_CHUNK: self._handle_get_chunk,
}
def test_file_size(self):
ds1 = DSStore('./storage/')
ret = ds1._file_size('test_chunk')
eq_(ret, 21)
ret = ds1._file_size('nosuchchunk')
assert ret < 0
def __init__(self, config, test=False):
"""@config: ConfigParser() object"""
self._config = config
ds_ip = config.get('dataserver', 'ds_ip')
ds_port = config.getint('dataserver', 'ds_port')
self._server = eventlet.listen((ds_ip, ds_port))
thread_num = config.getint('threads', 'thread_num')
self._pool = eventlet.GreenPool(thread_num)
store_dir = config.get('storage', 'chunk_store_dir')
logging.info('get store dir: %s', store_dir)
store_type = config.get('storage', 'type')
if store_type == DS_LOCAL:
self._ds = DSStore(store_dir)
elif store_type == DS_MEMORY:
self._ds = MemStore(store_dir)
else:
self._ds = DSStore(store_dir)
# register to mds when not in testmode
if not test:
mds_ip = config.get('mds', 'mds_ip')
mds_port = config.getint('mds', 'mds_port')
self._add_ds(mds_ip, mds_port)
# handler for request to ds
self._handlers = {
OP_ADD_CHUNK: self._handle_add_chunk,
OP_DELETE_CHUNK: self._handle_delete_chunk,
OP_GET_CHUNK: self._handle_get_chunk,
}
def test_write_chunk(self):
# correct write test in setup
data = 'data write into file!'
# incorrect write
ds2 = DSStore('./nosuchdir')
ret = ds2.write_chunk('0_obj2_chk2', data)
eq_(ret, RET_FAILURE)
def test_init(self):
ds1 = DSStore()
eq_(ds1.store_dir, './storage/')
ds2 = DSStore('./noslash')
eq_(ds2.store_dir, './noslash/')
ds3 = DSStore(' ./needstrip/ ')
eq_(ds3.store_dir, './needstrip/')
def test_info_chunk(self):
ds1 = DSStore('./storage/')
ret = ds1.info_chunk('test_chunk')
eq_(ret['state'], CHUNK_OK)
eq_(ret['size'], 21)
ret = ds1.info_chunk('nosuchchunk')
eq_(ret['state'], CHUNK_MISSING)
# assert_less(ret['size'], 0)
assert ret['size'] < 0
def test_remove_chunk(self):
# correct remove test in teardown
# incorret remove
ds1 = DSStore('./storage/')
ret = ds1.remove_chunk('nosuchchunk')
eq_(ret, RET_FAILURE)
ds2 = DSStore('./nosuchdir/')
ret = ds2.remove_chunk('nosuchchunk')
eq_(ret, RET_FAILURE)
def test_read_chunk(self):
ds1 = DSStore('./storage/')
ret = ds1.read_chunk('test_chunk', 3, [0, 1])
eq_(ret[0], RET_SUCCESS)
eq_(ret[1], ['data wr', 'ite int'])
ret = ds1.read_chunk('test_chunk', 3, [2, 1])
eq_(ret[0], RET_SUCCESS)
eq_(ret[1], ['o file!', 'ite int'])
ret = ds1.read_chunk('nosuchchunk', 3, [0, 1])
eq_(ret[0], RET_FAILURE)
eq_(ret[1], [])
ret = ds1.read_chunk('test_chunk', 0, [0, 1])
eq_(ret[0], RET_FAILURE)
class DSServer(object):
"""data server to handle network request and response"""
def __init__(self, config, test=False):
"""@config: ConfigParser() object"""
self._config = config
ds_ip = config.get('dataserver', 'ds_ip')
ds_port = config.getint('dataserver', 'ds_port')
self._server = eventlet.listen((ds_ip, ds_port))
thread_num = config.getint('threads', 'thread_num')
self._pool = eventlet.GreenPool(thread_num)
store_dir = config.get('storage', 'chunk_store_dir')
logging.info('get store dir: %s', store_dir)
store_type = config.get('storage', 'type')
if store_type == DS_LOCAL:
self._ds = DSStore(store_dir)
elif store_type == DS_MEMORY:
self._ds = MemStore(store_dir)
else:
self._ds = DSStore(store_dir)
# register to mds when not in testmode
if not test:
mds_ip = config.get('mds', 'mds_ip')
mds_port = config.getint('mds', 'mds_port')
self._add_ds(mds_ip, mds_port)
# handler for request to ds
self._handlers = {
OP_ADD_CHUNK: self._handle_add_chunk,
OP_DELETE_CHUNK: self._handle_delete_chunk,
OP_GET_CHUNK: self._handle_get_chunk,
}
def _add_ds(self, mds_ip='127.0.0.1', mds_port=8000):
"""register ds to mds"""
logging.info('add ds to mds')
rack_id = self._config.getint('dataserver', 'rack_id')
ds_ip = self._config.get('dataserver', 'ds_ip')
ds_port = self._config.getint('dataserver', 'ds_port')
packet = AddDSPacket(rack_id, ds_ip, ds_port)
msg = packet.get_message()
try:
sock = eventlet.connect((mds_ip, mds_port))
except socket.error:
logging.error('can not connect to mds %s:%d', mds_ip, mds_port)
sys.exit(
'can not connect to mds, start mds and set the conf file!')
sock_fd = sock.makefile('rw')
logging.info('add ds msg: %s', msg)
send_command(sock_fd, msg)
recv = recv_command(sock_fd)
state = recv['state']
if state == RET_FAILURE:
logging.error('add ds error, return :%s', recv)
sys.exit('add ds error, mds return ' + recv)
sock_fd.close()
return state
def _handle_add_chunk(self, filed, args):
"""handle client -> ds add chunk request, and response"""
logging.info('handle add chunk request')
# receive data from client
chunk_id = args['chunk_id']
data = recv_data(filed)
# write data to local filesystem
state = self._ds.write_chunk(chunk_id, data)
logging.info('add chunk: %s', chunk_id)
# reply to client
reply = AddChunkReplyPacket(state)
msg = reply.get_message()
logging.info("add chunk return: %s", msg)
send_command(filed, msg)
def _handle_delete_chunk(self, filed, args):
"""handle client -> ds delete chunk request, and response"""
logging.info('handle delete chunk request')
chunk_id = args['chunk_id']
# delete local filesystem chunk
state = self._ds.remove_chunk(chunk_id)
logging.info('add chunk: %s', chunk_id)
# reply to client
reply = DeleteChunkReplyPacket(state)
msg = reply.get_message()
logging.info("delete chunk return: %s", msg)
send_command(filed, msg)
def _handle_get_chunk(self, filed, args):
"""handle client -> ds get chunk requst, and response"""
logging.info('handle get chunk request')
chunk_id = args['chunk_id']
total = args['total']
lists = args['list']
logging.info('get chunk: %s', chunk_id)
# get data from local filesystem
state, data = self._ds.read_chunk(chunk_id, total, lists)
logging.info('read chunk return: %d', state)
# reply state
reply = GetChunkReplyPacket(state)
msg = reply.get_message()
logging.info("get chunk return: %s", msg)
send_command(filed, msg)
# reply data
if isinstance(data, list):
data = b''.join(data)
send_data(filed, data)
def _handle_conncetion(self, filed):
"""handle connected socket as a file"""
logging.info('connection start')
command = recv_command(filed)
self._handlers[command['method']](filed, command)
filed.close()
logging.info('disconnected')
def start(self):
"""
start data server
"""
while True:
try:
sock, address = self._server.accept()
logging.info('accepted %s:%s', address[0], address[1])
self._pool.spawn_n(self._handle_conncetion,
sock.makefile('rw'))
except (SystemExit, KeyboardInterrupt):
break
def teardown(self):
ds1 = DSStore('./storage/')
ret = ds1.remove_chunk('test_chunk')
eq_(ret, RET_SUCCESS)
def setup(self):
data = 'data write into file!'
ds1 = DSStore('./storage/')
ret = ds1.write_chunk('test_chunk', data)
eq_(ret, RET_SUCCESS)
class DSServer(object):
"""data server to handle network request and response"""
def __init__(self, config, test=False):
"""@config: ConfigParser() object"""
self._config = config
ds_ip = config.get('dataserver', 'ds_ip')
ds_port = config.getint('dataserver', 'ds_port')
self._server = eventlet.listen((ds_ip, ds_port))
thread_num = config.getint('threads', 'thread_num')
self._pool = eventlet.GreenPool(thread_num)
store_dir = config.get('storage', 'chunk_store_dir')
logging.info('get store dir: %s', store_dir)
store_type = config.get('storage', 'type')
if store_type == DS_LOCAL:
self._ds = DSStore(store_dir)
elif store_type == DS_MEMORY:
self._ds = MemStore(store_dir)
else:
self._ds = DSStore(store_dir)
# register to mds when not in testmode
if not test:
mds_ip = config.get('mds', 'mds_ip')
mds_port = config.getint('mds', 'mds_port')
self._add_ds(mds_ip, mds_port)
# handler for request to ds
self._handlers = {
OP_ADD_CHUNK: self._handle_add_chunk,
OP_DELETE_CHUNK: self._handle_delete_chunk,
OP_GET_CHUNK: self._handle_get_chunk,
}
def _add_ds(self, mds_ip='127.0.0.1', mds_port=8000):
"""register ds to mds"""
logging.info('add ds to mds')
rack_id = self._config.getint('dataserver', 'rack_id')
ds_ip = self._config.get('dataserver', 'ds_ip')
ds_port = self._config.getint('dataserver', 'ds_port')
packet = AddDSPacket(rack_id, ds_ip, ds_port)
msg = packet.get_message()
try:
sock = eventlet.connect((mds_ip, mds_port))
except socket.error:
logging.error('can not connect to mds %s:%d', mds_ip, mds_port)
sys.exit('can not connect to mds, start mds and set the conf file!')
sock_fd = sock.makefile('rw')
logging.info('add ds msg: %s', msg)
send_command(sock_fd, msg)
recv = recv_command(sock_fd)
state = recv['state']
if state == RET_FAILURE:
logging.error('add ds error, return :%s', recv)
sys.exit('add ds error, mds return ' + recv)
sock_fd.close()
return state
def _handle_add_chunk(self, filed, args):
"""handle client -> ds add chunk request, and response"""
logging.info('handle add chunk request')
# receive data from client
chunk_id = args['chunk_id']
data = recv_data(filed)
# write data to local filesystem
state = self._ds.write_chunk(chunk_id, data)
logging.info('add chunk: %s', chunk_id)
# reply to client
reply = AddChunkReplyPacket(state)
msg = reply.get_message()
logging.info("add chunk return: %s", msg)
send_command(filed, msg)
def _handle_delete_chunk(self, filed, args):
"""handle client -> ds delete chunk request, and response"""
logging.info('handle delete chunk request')
chunk_id = args['chunk_id']
# delete local filesystem chunk
state = self._ds.remove_chunk(chunk_id)
logging.info('add chunk: %s', chunk_id)
# reply to client
reply = DeleteChunkReplyPacket(state)
msg = reply.get_message()
logging.info("delete chunk return: %s", msg)
send_command(filed, msg)
def _handle_get_chunk(self, filed, args):
"""handle client -> ds get chunk requst, and response"""
logging.info('handle get chunk request')
chunk_id = args['chunk_id']
total = args['total']
lists = args['list']
logging.info('get chunk: %s', chunk_id)
# get data from local filesystem
state, data = self._ds.read_chunk(chunk_id, total, lists)
logging.info('read chunk return: %d', state)
# reply state
reply = GetChunkReplyPacket(state)
msg = reply.get_message()
logging.info("get chunk return: %s", msg)
send_command(filed, msg)
# reply data
if isinstance(data, list):
data = b''.join(data)
send_data(filed, data)
def _handle_conncetion(self, filed):
"""handle connected socket as a file"""
logging.info('connection start')
command = recv_command(filed)
self._handlers[command['method']](filed, command)
filed.close()
logging.info('disconnected')
def start(self):
"""
start data server
"""
while True:
try:
sock, address = self._server.accept()
logging.info('accepted %s:%s', address[0], address[1])
self._pool.spawn_n(self._handle_conncetion, sock.makefile('rw'))
except (SystemExit, KeyboardInterrupt):
break