def getRack():
try:
def watcher(event):
logger.info("/rack children changed, need update memory")
getRack()
zk_client.get('/rack', watcher)
children = zk_client.get_children('/rack')
for child in children:
rack_name = child.encode('utf-8')
RACK_STORE[rack_name] = []
path1 = join('/rack', rack_name)
children1 = zk_client.get_children(path1)
for child1 in children1:
store_id = child1.encode('utf-8')
RACK_STORE[rack_name].append(store_id)
path2 = join(path1, store_id)
data, stat = zk_client.get(path2)
if data:
parsed_data = json.loads(data)
ip = parsed_data['stat'].split(':')[0].encode('utf-8')
STORE_TO_IP[store_id] = ip
IP_TO_STORE[ip] = store_id
STORE_RACK[store_id] = rack_name
STORE_INFO[FREE_VOLUME_KEY+store_id] = -1
STORE_INFO[VOLUME_KEY+store_id] = 0
else:
logger.warn("getRack() called zk data is None path: %s", path2)
return False
return True
except Exception as ex:
logger.error("getRack() called error: %s", str(ex))
return False
def initFromStore(store_ip):
store_conn = genStoreConn(store_ip, config.store_stat_port)
url = "/info"
retcode, status, data = store_conn.request('GET', url)
if retcode and status >= 200 and status < 300:
store_data = json.loads(data)
if store_data['ret'] != 1:
return False
free_volumes_num = volumes_num = 0
free_volumes = store_data['free_volumes']
if free_volumes is not None:
free_volumes_num = len(free_volumes) - 1
STORE_INFO[FREE_VOLUME_KEY+IP_TO_STORE[store_ip]] = free_volumes_num
volumes = store_data['volumes']
if volumes is not None:
volumes_num = len(volumes)
STORE_INFO[VOLUME_KEY+IP_TO_STORE[store_ip]] = volumes_num
return True
logger.error("initFromStore() called failed: store_ip: %s", store_ip)
return False
def initFromStore(store_ip):
store_conn = genStoreConn(store_ip, config.store_stat_port)
url = "/info"
retcode, status, data = store_conn.request('GET', url)
if retcode and status >= 200 and status < 300:
store_data = json.loads(data)
if store_data['ret'] != 1:
return False
free_volumes_num = volumes_num = 0
free_volumes = store_data['free_volumes']
if free_volumes is not None:
free_volumes_num = len(free_volumes) - 1
STORE_INFO[FREE_VOLUME_KEY + IP_TO_STORE[store_ip]] = free_volumes_num
volumes = store_data['volumes']
if volumes is not None:
volumes_num = len(volumes)
STORE_INFO[VOLUME_KEY + IP_TO_STORE[store_ip]] = volumes_num
return True
logger.error("initFromStore() called failed: store_ip: %s", store_ip)
return False
def addGroupStore(group_id, store_id):
try:
if zk_client.exists('/group') is None:
zk_client.create('/group')
path = '/group/' + str(group_id)
if zk_client.exists(path) is None:
zk_client.create(path)
path1 = path + '/' + str(store_id)
if zk_client.exists(path1) is None:
zk_client.create(path1)
return True
except Exception as ex:
logger.error("addGroupStore() called error: %s", str(ex))
return False
def addGroupStore(group_id, store_id):
try:
if zk_client.exists('/group') is None:
zk_client.create('/group')
path = '/group/' + str(group_id)
if zk_client.exists(path) is None:
zk_client.create(path)
path1 = path + '/' + str(store_id)
if zk_client.exists(path1) is None:
zk_client.create(path1)
return True
except Exception as ex:
logger.error("addGroupStore() called error: %s", str(ex))
return False
def addVolumeStore(volume_id, store_id):
try:
if zk_client.exists("/volume") is None:
zk_client.create("/volume")
path = "/volume/" + str(volume_id)
if zk_client.exists(path) is None:
zk_client.create(path)
path1 = path + "/" + str(store_id)
if zk_client.exists(path1) is None:
zk_client.create(path1)
return True
except Exception as ex:
logger.error("addVolumeStore() called error: %s", str(ex))
return False
def storeAddVolume(store_ip, volume_id):
if volume_id <= 0:
return None
store_conn = genStoreConn(store_ip, config.store_admin_port)
url = "/add_volume"
value = {}
value['vid'] = volume_id
body = urllib.urlencode(value)
retcode, status, data = store_conn.request('POST', url, body, headers=genHttpHeaders())
if retcode and status >= 200 and status < 300:
return json.loads(data)
logger.error("storeAddVolume() called failed: status: %d, store_ip: %s, volume_id: %d", status, store_ip, volume_id)
return None
def storeAddVolume(store_ip, volume_id):
if volume_id <= 0:
return None
store_conn = genStoreConn(store_ip, config.store_admin_port)
url = "/add_volume"
value = {}
value['vid'] = volume_id
body = urllib.urlencode(value)
retcode, status, data = store_conn.request('POST', url, body, headers=genHttpHeaders())
if retcode and status >= 200 and status < 300:
return json.loads(data)
logger.error("storeAddVolume() called failed: status: %d, store_ip: %s, volume_id: %d", status, store_ip, volume_id)
return None
def initBfsData():
from zk_client import initFromZk
if initFromZk():
logger.info("initFromZk() is called, success")
else:
logger.error("initFromZk() is called, failed")
return False
from store_client import initFromStore
for store_ip in IP_TO_STORE:
if initFromStore(store_ip):
logger.info("initFromStore() is called, success")
else:
logger.info("initFromStore() is called, failed")
return False
logger.info("initBfsData() is called, success")
return True
def initBfsData():
from zk_client import initFromZk
if initFromZk():
logger.info("initFromZk() is called, success")
else:
logger.error("initFromZk() is called, failed")
return False
from store_client import initFromStore
for store_ip in IP_TO_STORE:
if initFromStore(store_ip):
logger.info("initFromStore() is called, success")
else:
logger.info("initFromStore() is called, failed")
return False
logger.info("initBfsData() is called, success")
return True
def storeAddFreeVolume(store_ip, base_dir, num_volumes):
if num_volumes <= 0:
return None
store_conn = genStoreConn(store_ip, config.store_admin_port)
url = "/add_free_volume"
value = {}
value['n'] = num_volumes
value['bdir'] = base_dir
value['idir'] = base_dir
body = urllib.urlencode(value)
retcode, status, data = store_conn.request('POST', url, body, headers=genHttpHeaders())
if retcode and status >= 200 and status < 300:
return json.loads(data)
logger.error("storeAddFreeVolume() called failed: store_ip: %s, num_volumes: %d base_dir: %s",
store_ip, num_volumes, base_dir)
return None
def storeAddFreeVolume(store_ip, base_dir, num_volumes):
if num_volumes <= 0:
return None
print store_ip
store_conn = genStoreConn(store_ip, config.store_admin_port)
url = "/add_free_volume"
value = {}
value['n'] = num_volumes
value['bdir'] = base_dir
value['idir'] = base_dir
body = urllib.urlencode(value)
retcode, status, data = store_conn.request('POST', url, body, headers=genHttpHeaders())
if retcode and status >= 200 and status < 300:
return json.loads(data)
logger.error("storeAddFreeVolume() called failed: store_ip: %s, num_volumes: %d base_dir: %s",
store_ip, num_volumes, base_dir)
return None
def initFromZk():
if getRack():
logger.info("getRack() called success")
else:
logger.error("getRack() called failed, need check now")
return False
if getAllVolume():
logger.info("getAllVolume() called success")
else:
logger.info("getAllVolume() called failed, need check now")
return False
if getAllGroup():
logger.info("getAllGroup() called success")
else:
logger.info("getAllGroup() called failed, need check now")
return False
return True
def initFromZk():
if getRack():
logger.info("getRack() called success")
else:
logger.error("getRack() called failed, need check now")
return False
if getAllVolume():
logger.info("getAllVolume() called success")
else:
logger.info("getAllVolume() called failed, need check now")
return False
if getAllGroup():
logger.info("getAllGroup() called success")
else:
logger.info("getAllGroup() called failed, need check now")
return False
return True
def getAllGroup():
try:
if zk_client.exists('/group') is None:
return True
children = zk_client.get_children('/group')
for child in children:
group_id = child
if int(group_id) > MAX_GROUP_ID:
MAX_GROUP_ID = int(group_id)
path1 = join('/group', group_id)
children1 = zk_client.get_children(path1)
for child1 in children1:
store_id = child1
STORE_GROUP[store_id] = group_id
if not GROUP_STORE.has_key(store_id):
GROUP_STORE[group_id] = []
GROUP_STORE[group_id].append(store_id)
return True
except Exception as ex:
logger.error("getAllGroup() called error: %s", str(ex))
return False
def getAllVolume():
global MAX_VOLUME_ID
try:
if zk_client.exists('/volume') is None:
return True
children = zk_client.get_children('/volume')
for child in children:
volume_id = child
if int(volume_id) > MAX_VOLUME_ID:
MAX_VOLUME_ID = int(volume_id)
path1 = join('/volume', volume_id)
children1 = zk_client.get_children(path1)
for child1 in children1:
store_id = child1
if not STORE_VOLUME.has_key(store_id):
STORE_VOLUME[store_id] = []
STORE_VOLUME[store_id].append(volume_id)
return True
except Exception as ex:
logger.error("getAllVolume() called error: %s", str(ex))
return False
def getAllVolume():
global MAX_VOLUME_ID
try:
if zk_client.exists('/volume') is None:
return True
children = zk_client.get_children('/volume')
for child in children:
volume_id = child
if int(volume_id) > MAX_VOLUME_ID:
MAX_VOLUME_ID = int(volume_id)
path1 = join('/volume', volume_id)
children1 = zk_client.get_children(path1)
for child1 in children1:
store_id = child1
if not STORE_VOLUME.has_key(store_id):
STORE_VOLUME[store_id] = []
STORE_VOLUME[store_id].append(volume_id)
return True
except Exception as ex:
logger.error("getAllVolume() called error: %s", str(ex))
return False
def getAllGroup():
global MAX_GROUP_ID
try:
if zk_client.exists('/group') is None:
return True
children = zk_client.get_children('/group')
for child in children:
group_id = child.encode('utf-8')
if int(group_id) > MAX_GROUP_ID:
MAX_GROUP_ID = int(group_id)
path1 = join('/group', group_id)
children1 = zk_client.get_children(path1)
for child1 in children1:
store_id = child1.encode('utf-8')
STORE_GROUP[store_id] = group_id
if not GROUP_STORE.has_key(group_id):
GROUP_STORE[group_id] = []
GROUP_STORE[group_id].append(store_id)
return True
except Exception as ex:
logger.error("getAllGroup() called error: %s", str(ex))
return False
return app
app = createApp()
def initBfsData():
from zk_client import initFromZk
if initFromZk():
logger.info("initFromZk() is called, success")
else:
logger.error("initFromZk() is called, failed")
return False
from store_client import initFromStore
for store_ip in IP_TO_STORE:
if initFromStore(store_ip):
logger.info("initFromStore() is called, success")
else:
logger.info("initFromStore() is called, failed")
return False
logger.info("initBfsData() is called, success")
return True
if not initBfsData():
logger.error("initBfsData() called, failed QUIT NOW")
import sys
sys.exit()
app = Flask(__name__)
return app
app = createApp()
def initBfsData():
from zk_client import initFromZk
if initFromZk():
logger.info("initFromZk() is called, success")
else:
logger.error("initFromZk() is called, failed")
return False
from store_client import initFromStore
for store_ip in IP_TO_STORE:
if initFromStore(store_ip):
logger.info("initFromStore() is called, success")
else:
logger.info("initFromStore() is called, failed")
return False
logger.info("initBfsData() is called, success")
return True
if not initBfsData():
logger.error("initBfsData() called, failed QUIT NOW")
import sys
sys.exit()