def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = PingRsp()
facts = bash_o('ceph -s -f json')
mon_facts = jsonobject.loads(facts)
found = False
for mon in mon_facts.monmap.mons:
if cmd.monAddr in mon.addr:
found = True
break
if not found:
rsp.success = False
rsp.failure = "MonAddrChanged"
rsp.error = 'The mon addr is changed on the mon server[uuid:%s], not %s anymore.' \
'Reconnect the ceph primary storage' \
' may solve this issue' % (cmd.monUuid, cmd.monAddr)
return jsonobject.dumps(rsp)
pool, objname = cmd.testImagePath.split('/')
create_img = shell.ShellCmd("echo zstack | rados -p '%s' put '%s' -" % (pool, objname))
create_img(False)
if create_img.return_code != 0:
rsp.success = False
rsp.failure = 'UnableToCreateFile'
rsp.error = "%s %s" % (create_img.stderr, create_img.stdout)
else:
shell.run("rados -p '%s' rm '%s'" % (pool, objname))
return jsonobject.dumps(rsp)
def get_images_metadata(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
valid_images_info = ""
bs_sftp_info_file = cmd.backupStoragePath + '/' + self.SFTP_METADATA_FILE
image_uuid_list = []
with open(bs_sftp_info_file) as fd:
images_info = fd.read()
for image_info in images_info.split('\n'):
if image_info != '':
image_json = jsonobject.loads(image_info)
# todo support multiple bs
image_uuid = image_json['uuid']
image_install_path = image_json["backupStorageRefs"][0]["installPath"]
if image_uuid in image_uuid_list:
logger.debug("duplicate uuid %s, ignore" % image_json["uuid"])
continue
image_uuid_list.append(image_uuid)
ret = bash_r("ls %s" % image_install_path)
if ret == 0 :
logger.info("Check image %s install path %s successfully!" % (image_uuid, image_install_path))
valid_images_info = image_info + '\n' + valid_images_info
else:
logger.warn("Image %s install path %s is invalid!" % (image_uuid, image_install_path))
rsp = GetImageMetaDataResponse()
rsp.imagesMetaData = valid_images_info
return jsonobject.dumps(rsp)
def get_images_metadata(self, req):
logger.debug("meilei: get images metadata")
cmd = jsonobject.loads(req[http.REQUEST_BODY])
pool_name = cmd.poolName
bs_uuid = pool_name.split("-")[-1]
valid_images_info = ""
self.get_metadata_file(bs_uuid, self.CEPH_METADATA_FILE)
last_image_install_path = ""
bs_ceph_info_file = "/tmp/%s" % self.CEPH_METADATA_FILE
with open(bs_ceph_info_file) as fd:
images_info = fd.read()
for image_info in images_info.split('\n'):
if image_info != '':
image_json = jsonobject.loads(image_info)
# todo support multiple bs
image_uuid = image_json['uuid']
image_install_path = image_json["backupStorageRefs"][0]["installPath"]
ret = bash_r("rbd info %s" % image_install_path.split("//")[1])
if ret == 0 :
logger.info("Check image %s install path %s successfully!" % (image_uuid, image_install_path))
if image_install_path != last_image_install_path:
valid_images_info = image_info + '\n' + valid_images_info
last_image_install_path = image_install_path
else:
logger.warn("Image %s install path %s is invalid!" % (image_uuid, image_install_path))
self.put_metadata_file(bs_uuid, self.CEPH_METADATA_FILE)
rsp = GetImageMetaDataResponse()
rsp.imagesMetadata= valid_images_info
return jsonobject.dumps(rsp)
def async_call_wait_for_complete(self, apicmd, exception_on_error=True, interval=500, fail_soon=False):
self._check_not_none_field(apicmd)
timeout = apicmd.timeout
if not timeout:
timeout = 1800000
cmd = {apicmd.FULL_NAME: apicmd}
logger.debug("async call[url: %s, request: %s]" % (self.api_url, jsonobject.dumps(cmd)))
jstr = http.json_dump_post(self.api_url, cmd, fail_soon=fail_soon)
rsp = jsonobject.loads(jstr)
if rsp.state == 'Done':
logger.debug("async call[url: %s, response: %s]" % (self.api_url, rsp.result))
reply = jsonobject.loads(rsp.result)
(name, event) = (reply.__dict__.items()[0])
if exception_on_error and not event.success:
raise ApiError('API call[%s] failed because %s' % (name, self._error_code_to_string(event.error)))
return name, event
curr = 0
finterval = float(float(interval) / float(1000))
ret_uuid = rsp.uuid
while rsp.state != 'Done' and curr < timeout:
time.sleep(finterval)
rsp = self._get_response(ret_uuid)
curr += interval
if curr >= timeout:
raise ApiError('API call[%s] timeout after %dms' % (apicmd.FULL_NAME, curr))
logger.debug("async call[url: %s, response: %s] after %dms" % (self.api_url, rsp.result, curr))
reply = jsonobject.loads(rsp.result)
(name, event) = (reply.__dict__.items()[0])
if exception_on_error and not event.success:
raise ApiError('API call[%s] failed because %s' % (name, self._error_code_to_string(event.error)))
return name, event
def create_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
spath = self._normalize_install_path(cmd.snapshotPath)
do_create = True
if cmd.skipOnExisting:
image_name, sp_name = spath.split('@')
o = shell.call('rbd --format json snap ls %s' % image_name)
o = jsonobject.loads(o)
for s in o:
if s.name_ == sp_name:
do_create = False
if do_create:
o = shell.ShellCmd('rbd snap create %s' % spath)
o(False)
if o.return_code != 0:
shell.run("rbd snap rm %s" % spath)
o.raise_error()
rsp = CreateSnapshotRsp()
rsp.size = self._get_file_size(spath)
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def init(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
o = shell.call('ceph mon_status')
mon_status = jsonobject.loads(o)
fsid = mon_status.monmap.fsid_
existing_pools = shell.call('ceph osd lspools')
for pool in cmd.pools:
if pool.predefined and pool.name not in existing_pools:
raise Exception('cannot find pool[%s] in the ceph cluster, you must create it manually' % pool.name)
elif pool.name not in existing_pools:
shell.call('ceph osd pool create %s 128' % pool.name)
rsp = InitRsp()
if cmd.nocephx is False:
o = shell.call("ceph -f json auth get-or-create client.zstack mon 'allow r' osd 'allow *' 2>/dev/null").strip(
' \n\r\t')
o = jsonobject.loads(o)
rsp.userKey = o[0].key_
rsp.fsid = fsid
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = PingRsp()
facts = bash_o('ceph -s -f json')
mon_facts = jsonobject.loads(facts)
found = False
for mon in mon_facts.monmap.mons:
if cmd.monAddr in mon.addr:
found = True
break
if not found:
rsp.success = False
rsp.failure = "MonAddrChanged"
rsp.error = 'The mon addr is changed on the mon server[uuid:%s], not %s anymore.' \
'Reconnect the ceph primary storage' \
' may solve this issue' % (cmd.monUuid, cmd.monAddr)
return jsonobject.dumps(rsp)
create_img = shell.ShellCmd('rbd create %s --image-format 2 --size 1' % cmd.testImagePath)
create_img(False)
if create_img.return_code != 0:
rsp.success = False
rsp.failure = 'UnableToCreateFile'
rsp.error = "%s %s" % (create_img.stderr, create_img.stdout)
else:
rm_img = shell.ShellCmd('rbd rm %s' % cmd.testImagePath)
rm_img(False)
return jsonobject.dumps(rsp)
def get_volume_snapinfos(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
vpath = self._normalize_install_path(cmd.volumePath)
ret = shell.call('rbd --format=json snap ls %s' % vpath)
rsp = GetVolumeSnapInfosRsp()
rsp.snapInfos = jsonobject.loads(ret)
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
facts = bash_o('ceph -s -f json')
mon_facts = jsonobject.loads(facts)
found = False
for mon in mon_facts.monmap.mons:
if cmd.monAddr in mon.addr:
found = True
break
rsp = PingRsp()
if not found:
rsp.success = False
rsp.failure = "MonAddrChanged"
rsp.error = 'The mon addr is changed on the mon server[uuid:%s], not %s anymore.' \
'Reconnect the ceph primary storage' \
' may solve this issue' % (cmd.monUuid, cmd.monAddr)
return jsonobject.dumps(rsp)
def retry(times=3, sleep_time=3):
def wrap(f):
@functools.wraps(f)
def inner(*args, **kwargs):
for i in range(0, times):
try:
return f(*args, **kwargs)
except Exception as e:
logger.error(e)
time.sleep(sleep_time)
rsp.error = ("Still failed after retry. Below is detail:\n %s" % e)
return inner
return wrap
@retry()
def doPing():
# try to delete test file, ignore the result
pool, objname = cmd.testImagePath.split('/')
bash_r("rados -p '%s' rm '%s'" % (pool, objname))
r, o, e = bash_roe("echo zstack | timeout 60 rados -p '%s' put '%s' -" % (pool, objname))
if r != 0:
rsp.success = False
rsp.failure = "UnableToCreateFile"
if r == 124:
# timeout happened
rsp.error = 'failed to create heartbeat object on ceph, timeout after 60s, %s %s' % (e, o)
raise Exception(rsp.error)
else:
rsp.error = "%s %s" % (e, o)
doPing()
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def get_facts(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
o = shell.call('ceph mon_status')
mon_status = jsonobject.loads(o)
fsid = mon_status.monmap.fsid_
rsp = GetFactsRsp()
rsp.fsid = fsid
return jsonobject.dumps(rsp)
def download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
pool, image_name = self._parse_install_path(cmd.installPath)
tmp_image_name = 'tmp-%s' % image_name
if cmd.url.startswith('http://') or cmd.url.startswith('https://'):
shell.call('set -o pipefail; wget --no-check-certificate -q -O - %s | rbd import --image-format 2 - %s/%s'
% (cmd.url, pool, tmp_image_name))
actual_size = linux.get_file_size_by_http_head(cmd.url)
elif cmd.url.startswith('file://'):
src_path = cmd.url.lstrip('file:')
src_path = os.path.normpath(src_path)
if not os.path.isfile(src_path):
raise Exception('cannot find the file[%s]' % src_path)
shell.call("rbd import --image-format 2 %s %s/%s" % (src_path, pool, tmp_image_name))
actual_size = os.path.getsize(src_path)
else:
raise Exception('unknown url[%s]' % cmd.url)
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
_1()
file_format = shell.call("set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (pool, tmp_image_name, pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name))
o = shell.call('rbd --format json info %s/%s' % (pool, image_name))
image_stats = jsonobject.loads(o)
rsp = DownloadRsp()
rsp.size = long(image_stats.size_)
rsp.actualSize = actual_size
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def sync_call(self, apicmd, exception_on_error=True, fail_soon=False):
self._check_not_none_field(apicmd)
cmd = {apicmd.FULL_NAME: apicmd}
logger.debug("sync_call[url: %s, request: %s]" % (self.api_url, jsonobject.dumps(cmd)))
jstr = http.json_dump_post(self.api_url, cmd, fail_soon=fail_soon)
logger.debug("sync_call[url: %s, response: %s]" % (self.api_url, jstr))
rsp = jsonobject.loads(jstr)
reply = jsonobject.loads(rsp.result)
(name, r) = reply.__dict__.items()[0]
if exception_on_error:
if not r.success:
raise ApiError('API call[%s] failed because %s' % (name, self._error_code_to_string(r.error)))
return name, r
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
path = self._normalize_install_path(cmd.installPath)
o = shell.call('rbd snap ls --format json %s' % path)
o = jsonobject.loads(o)
if len(o) > 0:
raise Exception('unable to delete %s; the volume still has snapshots' % cmd.installPath)
shell.call('rbd rm %s' % path)
rsp = AgentResponse()
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def add_dhcp_entry(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
entries = []
gateways = []
for e in cmd.dhcpEntries:
entry = DhcpEntry.from_dhcp_info(e)
entries.append(entry)
gateways.append(entry.gateway)
if cmd.rebuild:
self._rebuild_all(entries)
else:
self._merge(entries)
rsp = AddDhcpEntryRsp()
try:
if self._add_dhcp_range_if_need(gateways):
self._restart_dnsmasq()
else:
self._refresh_dnsmasq()
except virtualrouter.VirtualRouterError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
def _set_capacity_to_response(self, rsp):
o = shell.call('ceph df -f json')
df = jsonobject.loads(o)
if df.stats.total_bytes__ is not None:
total = long(df.stats.total_bytes_)
elif df.stats.total_space__ is not None:
total = long(df.stats.total_space__) * 1024
else:
raise Exception('unknown ceph df output: %s' % o)
if df.stats.total_avail_bytes__ is not None:
avail = long(df.stats.total_avail_bytes_)
elif df.stats.total_avail__ is not None:
avail = long(df.stats.total_avail_) * 1024
else:
raise Exception('unknown ceph df output: %s' % o)
rsp.totalCapacity = total
rsp.availableCapacity = avail
rsp.xsky = isXsky()
if not df.pools:
return
pools = ceph.getCephPoolsCapacity()
if not pools:
return
rsp.poolCapacities = []
for pool in pools:
poolCapacity = CephPoolCapacity(pool.poolName, pool.availableCapacity, pool.replicatedSize, pool.usedCapacity, pool.poolTotalSize)
rsp.poolCapacities.append(poolCapacity)
def downloadfromimagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.imagestore_client.download_from_imagestore(cmd.cacheDir, cmd.hostname, cmd.backupStorageInstallPath,
cmd.primaryStorageInstallPath)
rsp = AliyunNasResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
def remove_dhcp_entry(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RemoveDhcpEntryRsp()
try:
for e in cmd.dhcpEntries:
net_dev = shell.call("ifconfig|grep -i %s|awk '{print $1}'" % e.vrNicMac)
net_dev = net_dev.strip('\t\r\n ')
mac2 = e.mac.replace(':', '')
shell.call("sed -i '/%s/d' %s; \
sed -i '/^$/d' %s; \
sed -i '/%s/d' %s; \
sed -i '/^$/d' %s; \
sed -i '/%s/d' %s; \
sed -i '/^$/d' %s; \
dhcp_release %s %s %s"\
% (e.mac, self.HOST_DHCP_FILE, \
self.HOST_DHCP_FILE, \
mac2, self.HOST_OPTION_FILE, \
self.HOST_OPTION_FILE, \
e.ip, self.HOST_DNS_FILE, \
self.HOST_DNS_FILE, \
net_dev, e.ip, e.mac))
except virtualrouter.VirtualRouterError as e:
logger.warn(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
def delete(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
for to in cmd.lbs:
self._kill_lb(to)
rsp = virtualrouter.AgentResponse()
return jsonobject.dumps(rsp)
def deletebits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AliyunNasResponse()
self.delNasBits(cmd.folder, cmd.path)
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.uuid)
return jsonobject.dumps(rsp)
def remove_eip(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = RemoveEipRsp()
self._remove_eip(cmd.eip)
return jsonobject.dumps(rsp)
def sync_eip(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = SyncEipRsp()
def remove_eip_chain(table):
for c in table.children:
if c.name.startswith('eip-'):
c.delete()
ipt = iptables.from_iptables_save()
nat = ipt.get_table(ipt.NAT_TABLE_NAME)
if nat:
remove_eip_chain(nat)
filter_table = ipt.get_table(ipt.FILTER_TABLE_NAME)
if filter_table:
remove_eip_chain(filter_table)
ipt.iptable_restore()
try:
for eip in cmd.eips:
self._create_eip(eip)
except virtualrouter.VirtualRouterError as e:
logger.warning(linux.get_exception_stacktrace())
rsp.error = str(e)
rsp.success = False
return jsonobject.dumps(rsp)
def unprotect_snapshot(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
spath = self._normalize_install_path(cmd.snapshotPath)
shell.call('rbd snap unprotect %s' % spath)
return jsonobject.dumps(AgentResponse())
def download_from_sftp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DownloadBitsFromSftpBackupStorageRsp()
sub_vol_dir = os.path.dirname(cmd.primaryStorageInstallPath)
if not os.path.exists(sub_vol_dir):
parent_dir = os.path.dirname(sub_vol_dir)
shell.call('mkdir -p %s' % parent_dir)
shell.call('btrfs subvolume create %s' % sub_vol_dir)
linux.scp_download(cmd.hostname, cmd.sshKey, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath)
def get_image_format():
out = shell.call('qemu-img info %s' % cmd.primaryStorageInstallPath)
for l in out.split('\n'):
if 'file format' in l:
_, f = l.split(':')
return f.strip()
raise Exception('cannot get image format of %s, qemu-img info outputs:\n%s\n' % (cmd.primaryStorageInstallPath, out))
f = get_image_format()
if 'qcow2' in f:
shell.call('/usr/bin/qemu-img convert -f qcow2 -O raw %s %s.img' % (cmd.primaryStorageInstallPath, cmd.primaryStorageInstallPath))
shell.call('mv %s.img %s' % (cmd.primaryStorageInstallPath, cmd.primaryStorageInstallPath))
elif 'raw' in f:
pass
else:
raise Exception('unsupported image format[%s] of %s' % (f, cmd.primaryStorageInstallPath))
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
logger.debug('downloaded %s:%s to %s' % (cmd.hostname, cmd.backupStorageInstallPath, cmd.primaryStorageInstallPath))
return jsonobject.dumps(rsp)
def create(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
path = self._normalize_install_path(cmd.installPath)
rsp = CreateEmptyVolumeRsp()
call_string = None
if isXsky():
# do NOT round to MB
call_string = 'rbd create --size %dB --image-format 2 %s' % (cmd.size, path)
rsp.size = cmd.size
else:
size_M = sizeunit.Byte.toMegaByte(cmd.size) + 1
call_string = 'rbd create --size %s --image-format 2 %s' % (size_M, path)
rsp.size = cmd.size + sizeunit.MegaByte.toByte(1)
if cmd.shareable:
call_string = call_string + " --image-shared"
skip_cmd = "rbd info %s ||" % path if cmd.skipIfExisting else ""
shell.call(skip_cmd + call_string)
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def purge_snapshots(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
vpath = self._normalize_install_path(cmd.volumePath)
shell.call('rbd snap purge %s' % vpath)
rsp = AgentResponse()
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def cp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
src_path = self._normalize_install_path(cmd.srcPath)
dst_path = self._normalize_install_path(cmd.dstPath)
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "CephCpVolume"
_, PFILE = tempfile.mkstemp()
stage = (cmd.threadContext['task-stage'], "10-90")[cmd.threadContext['task-stage'] is None]
def _get_progress(synced):
if not Report.url:
return synced
logger.debug("getProgress in ceph-agent")
percent = shell.call("tail -1 %s | grep -o '1\?[0-9]\{1,2\}%%' | tail -1" % PFILE).strip(' \t\n\r%')
if percent and Report.url:
report.progress_report(get_exact_percent(percent, stage), "report")
return synced
_, _, err = bash_progress_1('rbd cp %s %s 2> %s' % (src_path, dst_path, PFILE), _get_progress)
if os.path.exists(PFILE):
os.remove(PFILE)
if err:
raise err
rsp = CpRsp()
rsp.size = self._get_file_size(dst_path)
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def migrate_volume_segment(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
src_install_path = self._normalize_install_path(cmd.srcInstallPath)
dst_install_path = self._normalize_install_path(cmd.dstInstallPath)
src_size = self._get_file_size(src_install_path)
dst_size = self._get_dst_volume_size(dst_install_path, cmd.dstMonHostname, cmd.dstMonSshUsername, cmd.dstMonSshPassword, cmd.dstMonSshPort)
if dst_size > src_size:
if cmd.isXsky:
# xsky / ceph -> xsky, size must be equal
rsp.success = False
rsp.error = "Failed to migrate volume segment because dst size: %s > src size: %s" % (dst_size, src_size)
return jsonobject.dumps(rsp)
elif isXsky() == False:
# ceph -> ceph, don't check size
rsp.success = True
else:
# xsky -> ceph, not supported
rsp.success = False
rsp.error = "Failed to migrate volume segment because xsky migrate to ceph is not supported now"
return jsonobject.dumps(rsp)
if dst_size < src_size:
ret = self._resize_dst_volume(dst_install_path, src_size, cmd.dstMonHostname, cmd.dstMonSshUsername, cmd.dstMonSshPassword, cmd.dstMonSshPort)
if ret != 0:
rsp.success = False
rsp.error = "Failed to resize volume before migrate."
return jsonobject.dumps(rsp)
ret = self._migrate_volume_segment(cmd.parentUuid, cmd.resourceUuid, cmd.srcInstallPath, cmd.dstInstallPath, cmd.dstMonHostname, cmd.dstMonSshUsername, cmd.dstMonSshPassword, cmd.dstMonSshPort)
if ret != 0:
rsp.success = False
rsp.error = "Failed to migrate volume segment from one ceph primary storage to another."
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def get_volume_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
path = self._normalize_install_path(cmd.installPath)
rsp = GetVolumeSizeRsp()
rsp.size = self._get_file_size(path)
rsp.actualSize = self._get_file_actual_size(path)
return jsonobject.dumps(rsp)
def download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
pool, image_name = self._parse_install_path(cmd.installPath)
tmp_image_name = 'tmp-%s' % image_name
lichbd_file = os.path.join(pool, image_name)
tmp_lichbd_file = os.path.join(pool, tmp_image_name)
lichbd.lichbd_mkpool(os.path.dirname(lichbd_file))
shell.call('set -o pipefail; wget --no-check-certificate -q -O - %s | lichbd import - %s -p lichbd' % (cmd.url, tmp_lichbd_file))
@rollbackable
def _1():
if lichbd.lichbd_file_exist(tmp_lichbd_file):
lichbd.lichbd_rm(tmp_lichbd_file)
lichbd.lichbd_rm(lichbd_file)
_1()
qemu_img = lichbd.lichbd_get_qemu_img_path()
file_format = shell.call("set -o pipefail;%s info rbd:%s/%s 2>/dev/null | grep 'file format' | cut -d ':' -f 2" % (qemu_img, pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
lichbd.lichbd_mv(lichbd_file, tmp_lichbd_file)
size = lichbd.lichbd_file_size(lichbd_file)
rsp = DownloadRsp()
rsp.size = size
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def test_json(self):
b = B()
jstr = jsonobject.dumps(b)
nb = jsonobject.loads(jstr)
self.assertEqual(2, len(nb.cs))
self.assertEqual(3, len(nb.lst))
self.assertEqual(1, nb.lst[0])
self.assertEqual('home', nb.address)
self.assertEqual(19, nb.c.age)
self.assertEqual('hello', nb.name)
self.assertFalse(nb.male)
jstr2 = jsonobject.dumps(nb)
self.assertEqual(jstr, jstr2)
jb = jsonobject.loads(jstr)
print jb.xxxxx
print jb.lst
def convert_qcow2_to_raw(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.convert_image_raw(cmd)
def check_bits(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = CheckBitsRsp()
rsp.existing = os.path.exists(cmd.path)
return jsonobject.dumps(rsp)
def get_physical_capacity(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
def get_image_size(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetImageSizeRsp()
rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(
cmd.installPath)
return jsonobject.dumps(rsp)
def list(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ListResponse()
rsp.paths = kvmagent.listPath(cmd.path)
return jsonobject.dumps(rsp)
def commit_to_imagestore(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
return self.imagestore_client.commit_to_imagestore(cmd, req)
def apply_userdata(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self._apply_userdata_xtables(cmd.userdata)
self._apply_userdata_vmdata(cmd.userdata)
self._apply_userdata_restart_httpd(cmd.userdata)
return jsonobject.dumps(ApplyUserdataRsp())
def start_collectd_exporter(self, req):
@in_bash
def start_exporter(cmd):
conf_path = os.path.join(os.path.dirname(cmd.binaryPath),
'collectd.conf')
conf = '''Interval {{INTERVAL}}
FQDNLookup false
LoadPlugin syslog
LoadPlugin aggregation
LoadPlugin cpu
LoadPlugin disk
LoadPlugin interface
LoadPlugin memory
LoadPlugin network
LoadPlugin virt
<Plugin aggregation>
<Aggregation>
#Host "unspecified"
Plugin "cpu"
#PluginInstance "unspecified"
Type "cpu"
#TypeInstance "unspecified"
GroupBy "Host"
GroupBy "TypeInstance"
CalculateNum false
CalculateSum false
CalculateAverage true
CalculateMinimum false
CalculateMaximum false
CalculateStddev false
</Aggregation>
</Plugin>
<Plugin cpu>
ReportByCpu true
ReportByState true
ValuesPercentage true
</Plugin>
<Plugin disk>
Disk "/^sd[a-z]$/"
Disk "/^hd[a-z]$/"
Disk "/^vd[a-z]$/"
IgnoreSelected false
</Plugin>
<Plugin "interface">
{% for i in INTERFACES -%}
Interface "{{i}}"
{% endfor -%}
IgnoreSelected false
</Plugin>
<Plugin memory>
ValuesAbsolute true
ValuesPercentage false
</Plugin>
<Plugin virt>
Connection "qemu:///system"
RefreshInterval {{INTERVAL}}
HostnameFormat name
PluginInstanceFormat name
BlockDevice "/:hd[a-z]/"
IgnoreSelected true
</Plugin>
<Plugin network>
Server "localhost" "25826"
</Plugin>
'''
tmpt = Template(conf)
conf = tmpt.render({
'INTERVAL': cmd.interval,
'INTERFACES': interfaces,
})
need_restart_collectd = False
if os.path.exists(conf_path):
with open(conf_path, 'r') as fd:
old_conf = fd.read()
if old_conf != conf:
with open(conf_path, 'w') as fd:
fd.write(conf)
need_restart_collectd = True
else:
with open(conf_path, 'w') as fd:
fd.write(conf)
need_restart_collectd = True
cpid = linux.find_process_by_cmdline(['collectd', conf_path])
mpid = linux.find_process_by_cmdline(['collectdmon', conf_path])
if not cpid:
bash_errorout('collectdmon -- -C %s' % conf_path)
else:
if need_restart_collectd:
if not mpid:
bash_errorout('kill -TERM %s' % cpid)
bash_errorout('collectdmon -- -C %s' % conf_path)
else:
bash_errorout('kill -HUP %s' % mpid)
pid = linux.find_process_by_cmdline([cmd.binaryPath])
if not pid:
EXPORTER_PATH = cmd.binaryPath
LOG_FILE = os.path.join(os.path.dirname(EXPORTER_PATH),
cmd.binaryPath + '.log')
ARGUMENTS = cmd.startupArguments
if not ARGUMENTS:
ARGUMENTS = ""
bash_errorout('chmod +x {{EXPORTER_PATH}}')
bash_errorout(
"nohup {{EXPORTER_PATH}} {{ARGUMENTS}} >{{LOG_FILE}} 2>&1 < /dev/null &\ndisown"
)
para = jsonobject.loads(req[http.REQUEST_BODY])
rsp = kvmagent.AgentResponse()
eths = bash_o("ls /sys/class/net").split()
interfaces = []
for eth in eths:
eth = eth.strip(' \t\n\r')
if eth == 'lo': continue
if eth == 'bonding_masters': continue
elif eth.startswith('vnic'): continue
elif eth.startswith('outer'): continue
elif eth.startswith('br_'): continue
elif not eth: continue
else:
interfaces.append(eth)
for cmd in para.cmds:
start_exporter(cmd)
self.install_iptables()
return jsonobject.dumps(rsp)
def release_userdata(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
html_folder = os.path.join(self.USERDATA_ROOT, cmd.namespaceName,
'html', cmd.vmIp)
shell.call('rm -rf %s' % html_folder)
return jsonobject.dumps(ReleaseUserdataRsp())
def apply_dhcp(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
namespace_dhcp = {}
for d in cmd.dhcp:
lst = namespace_dhcp.get(d.namespaceName)
if not lst:
lst = []
namespace_dhcp[d.namespaceName] = lst
lst.append(d)
@in_bash
def apply(dhcp):
bridge_name = dhcp[0].bridgeName
namespace_name = dhcp[0].namespaceName
conf_file_path, dhcp_path, dns_path, option_path, log_path = self._make_conf_path(
namespace_name)
conf_file = '''\
domain-needed
bogus-priv
no-hosts
addn-hosts={{dns}}
dhcp-option=vendor:MSFT,2,1i
dhcp-lease-max=65535
dhcp-hostsfile={{dhcp}}
dhcp-optsfile={{option}}
log-facility={{log}}
interface={{iface_name}}
except-interface=lo
bind-interfaces
leasefile-ro
{% for g in gateways -%}
dhcp-range={{g}},static
{% endfor -%}
'''
br_num = shell.call(
"ip netns list-id | grep -w %s | awk '{print $2}'" %
namespace_name)
br_num = br_num.strip(' \t\r\n')
if not br_num:
raise Exception('cannot find the ID for the namespace[%s]' %
namespace_name)
tmpt = Template(conf_file)
conf_file = tmpt.render({
'dns':
dns_path,
'dhcp':
dhcp_path,
'option':
option_path,
'log':
log_path,
'iface_name':
'inner%s' % br_num,
'gateways': [d.gateway for d in dhcp if d.gateway]
})
restart_dnsmasq = cmd.rebuild
if not os.path.exists(conf_file_path) or cmd.rebuild:
with open(conf_file_path, 'w') as fd:
fd.write(conf_file)
else:
with open(conf_file_path, 'r') as fd:
c = fd.read()
if c != conf_file:
logger.debug(
'dnsmasq configure file for bridge[%s] changed, restart it'
% bridge_name)
restart_dnsmasq = True
with open(conf_file_path, 'w') as fd:
fd.write(conf_file)
logger.debug(
'wrote dnsmasq configure file for bridge[%s]\n%s' %
(bridge_name, conf_file))
info = []
for d in dhcp:
dhcp_info = {'tag': d.mac.replace(':', '')}
dhcp_info.update(d.__dict__)
dhcp_info['dns'] = ','.join(d.dns)
routes = []
# add classless-static-route (option 121) for gateway:
if d.isDefaultL3Network:
routes.append(','.join(['0.0.0.0/0', d.gateway]))
for route in d.hostRoutes:
routes.append(','.join([route.prefix, route.nexthop]))
dhcp_info['routes'] = ','.join(routes)
info.append(dhcp_info)
if not cmd.rebuild:
self._erase_configurations(d.mac, d.ip, dhcp_path,
dns_path, option_path)
dhcp_conf = '''\
{% for d in dhcp -%}
{% if d.isDefaultL3Network -%}
{{d.mac}},set:{{d.tag}},{{d.ip}},{{d.hostname}},infinite
{% else -%}
{{d.mac}},set:{{d.tag}},{{d.ip}},infinite
{% endif -%}
{% endfor -%}
'''
tmpt = Template(dhcp_conf)
dhcp_conf = tmpt.render({'dhcp': info})
mode = 'a+'
if cmd.rebuild:
mode = 'w'
with open(dhcp_path, mode) as fd:
fd.write(dhcp_conf)
option_conf = '''\
{% for o in options -%}
{% if o.isDefaultL3Network -%}
{% if o.gateway -%}
tag:{{o.tag}},option:router,{{o.gateway}}
{% endif -%}
{% if o.dns -%}
tag:{{o.tag}},option:dns-server,{{o.dns}}
{% endif -%}
{% if o.dnsDomain -%}
tag:{{o.tag}},option:domain-name,{{o.dnsDomain}}
{% endif -%}
{% if o.routes -%}
tag:{{o.tag}},option:classless-static-route,{{o.routes}}
{% endif -%}
{% else -%}
tag:{{o.tag}},3
tag:{{o.tag}},6
{% endif -%}
tag:{{o.tag}},option:netmask,{{o.netmask}}
{% if o.mtu -%}
tag:{{o.tag}},option:mtu,{{o.mtu}}
{% endif -%}
{% endfor -%}
'''
tmpt = Template(option_conf)
option_conf = tmpt.render({'options': info})
with open(option_path, mode) as fd:
fd.write(option_conf)
hostname_conf = '''\
{% for h in hostnames -%}
{% if h.isDefaultL3Network and h.hostname -%}
{{h.ip}} {{h.hostname}}
{% endif -%}
{% endfor -%}
'''
tmpt = Template(hostname_conf)
hostname_conf = tmpt.render({'hostnames': info})
with open(dns_path, mode) as fd:
fd.write(hostname_conf)
if restart_dnsmasq:
self._restart_dnsmasq(namespace_name, conf_file_path)
else:
self._refresh_dnsmasq(namespace_name, conf_file_path)
for k, v in namespace_dhcp.iteritems():
apply(v)
rsp = ApplyDhcpRsp()
return jsonobject.dumps(rsp)
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
report.resourceUuid = cmd.uuid
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
start = 10
end = 90
if cmd.stage:
start, end = get_scale(cmd.stage)
total = 0
for path in set(chain):
total = total + os.path.getsize(path)
written = 0
def _get_progress(synced):
logger.debug(
"getProgress in localstorage-agent, synced: %s, total: %s" %
(synced, total))
if not os.path.exists(PFILE):
return synced
fpread = open(PFILE, 'r')
lines = fpread.readlines()
if not lines:
fpread.close()
return synced
last = str(lines[-1]).strip().split('\r')[-1]
if not last or len(last.split()) < 1:
fpread.close()
return synced
line = last.split()[0]
if not line.isdigit():
return synced
if total > 0:
synced = long(line)
if synced < total:
percent = int(
round(
float(written + synced) / float(total) *
(end - start) + start))
report.progress_report(percent, "report")
synced = written
fpread.close()
return synced
for path in set(chain):
PATH = path
PASSWORD = cmd.dstPassword
USER = cmd.dstUsername
IP = cmd.dstIp
PORT = (cmd.dstPort and cmd.dstPort or "22")
DIR = os.path.dirname(path)
if cmd.dstUsername == 'root':
_, _, err = bash_progress_1(
# Fixes ZSTAC-13430: handle extremely complex password like ~ ` !@#$%^&*()_+-=[]{}|?<>;:'"/ .
'echo \'{{PASSWORD}}\' > /tmp/tmp_passwd && rsync -av --progress --relative {{PATH}} --rsh="/usr/bin/sshpass -f/tmp/tmp_passwd ssh -o StrictHostKeyChecking=no -p {{PORT}} -l {{USER}}" {{IP}}:/ 1>{{PFILE}}',
_get_progress,
False)
if err:
raise Exception('fail to migrate vm to host, because %s' %
str(err))
else:
raise Exception("cannot support migrate to non-root user host")
written += os.path.getsize(path)
bash_errorout(
'/usr/bin/sshpass -f/tmp/tmp_passwd ssh -o StrictHostKeyChecking=no -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"'
)
percent = int(
round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
def ping(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
facts = bash_o('ceph -s -f json')
mon_facts = jsonobject.loads(facts)
found = False
for mon in mon_facts.monmap.mons:
if cmd.monAddr in mon.addr:
found = True
break
rsp = PingRsp()
if not found:
rsp.success = False
rsp.failure = "MonAddrChanged"
rsp.error = 'The mon addr is changed on the mon server[uuid:%s], not %s anymore.' \
'Reconnect the ceph primary storage' \
' may solve this issue' % (cmd.monUuid, cmd.monAddr)
return jsonobject.dumps(rsp)
def retry(times=3, sleep_time=3):
def wrap(f):
@functools.wraps(f)
def inner(*args, **kwargs):
for i in range(0, times):
try:
return f(*args, **kwargs)
except Exception as e:
logger.error(e)
time.sleep(sleep_time)
rsp.error = (
"Still failed after retry. Below is detail:\n %s" % e)
return inner
return wrap
@retry()
def doPing():
# try to delete test file, ignore the result
pool, objname = cmd.testImagePath.split('/')
bash_r("rados -p '%s' rm '%s'" % (pool, objname))
r, o, e = bash_roe(
"echo zstack | timeout 60 rados -p '%s' put '%s' -" %
(pool, objname))
if r != 0:
rsp.success = False
rsp.failure = "UnableToCreateFile"
if r == 124:
# timeout happened
rsp.error = 'failed to create heartbeat object on ceph, timeout after 60s, %s %s' % (
e, o)
raise Exception(rsp.error)
else:
rsp.error = "%s %s" % (e, o)
doPing()
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def setup_self_fencer(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self.run_filesystem_fencer = True
@thread.AsyncThread
def heartbeat_file_fencer(heartbeat_file_path, ps_uuid):
try:
failure = 0
while self.run_filesystem_fencer:
time.sleep(cmd.interval)
touch = shell.ShellCmd(
'timeout %s touch %s; exit $?' %
(cmd.storageCheckerTimeout, heartbeat_file_path))
touch(False)
if touch.return_code == 0:
failure = 0
continue
logger.warn(
'unable to touch %s, %s %s' %
(heartbeat_file_path, touch.stderr, touch.stdout))
failure += 1
if failure == cmd.maxAttempts:
logger.warn(
'failed to touch the heartbeat file[%s] %s times, we lost the connection to the storage,'
'shutdown ourselves' %
(heartbeat_file_path, cmd.maxAttempts))
mountPath = (os.path.split(heartbeat_file_path))[0]
self.report_storage_status([ps_uuid], 'Disconnected')
kill_vm(cmd.maxAttempts, mountPath, True)
logger.debug('stop heartbeat[%s] for filesystem self-fencer' %
heartbeat_file_path)
except:
content = traceback.format_exc()
logger.warn(content)
gateway = cmd.storageGateway
if not gateway:
gateway = linux.get_gateway_by_default_route()
@thread.AsyncThread
def storage_gateway_fencer(gw):
failure = 0
try:
while self.run_filesystem_fencer:
time.sleep(cmd.interval)
ping = shell.ShellCmd(
"nmap -sP -PI %s | grep 'Host is up'" % gw)
ping(False)
if ping.return_code == 0:
failure = 0
continue
logger.warn(
'unable to ping the storage gateway[%s], %s %s' %
(gw, ping.stderr, ping.stdout))
failure += 1
if failure == cmd.maxAttempts:
logger.warn(
'failed to ping storage gateway[%s] %s times, we lost connection to the storage,'
'shutdown ourselves' % (gw, cmd.maxAttempts))
self.report_storage_status(cmd.psUuids, 'Disconnected')
kill_vm(cmd.maxAttempts)
logger.debug(
'stop gateway[%s] fencer for filesystem self-fencer' % gw)
except:
content = traceback.format_exc()
logger.warn(content)
for mount_point, uuid in zip(cmd.mountPoints, cmd.uuids):
if not linux.timeout_isdir(mount_point):
raise Exception('the mount point[%s] is not a directory' %
mount_point)
hb_file = os.path.join(
mount_point, 'heartbeat-file-kvm-host-%s.hb' % cmd.hostUuid)
heartbeat_file_fencer(hb_file, uuid)
if gateway:
storage_gateway_fencer(gateway)
else:
logger.warn(
'cannot find storage gateway, unable to setup storage gateway fencer'
)
return jsonobject.dumps(AgentRsp())
def write_image_metadata(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
meta_data = cmd.metaData
self._write_image_metadata(meta_data.installPath, meta_data)
rsp = WriteImageMetaDataResponse()
return jsonobject.dumps(rsp)
def setup_self_fencer(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
@thread.AsyncThread
def heartbeat_file_fencer(mount_path, ps_uuid, mounted_by_zstack):
def try_remount_fs():
if mount_path_is_nfs(mount_path):
shell.run("systemctl start nfs-client.target")
while self.run_filesystem_fencer(ps_uuid, created_time):
if linux.is_mounted(
path=mount_path) and touch_heartbeat_file():
self.report_storage_status([ps_uuid], 'Connected')
logger.debug(
"fs[uuid:%s] is reachable again, report to management"
% ps_uuid)
break
try:
logger.debug(
'fs[uuid:%s] is unreachable, it will be remounted after 180s'
% ps_uuid)
time.sleep(180)
if not self.run_filesystem_fencer(
ps_uuid, created_time):
break
linux.remount(url, mount_path, options)
self.report_storage_status([ps_uuid], 'Connected')
logger.debug(
"remount fs[uuid:%s] success, report to management"
% ps_uuid)
break
except:
logger.warn(
'remount fs[uuid:%s] fail, try again soon' %
ps_uuid)
kill_progresses_using_mount_path(mount_path)
logger.debug('stop remount fs[uuid:%s]' % ps_uuid)
def after_kill_vm():
if not killed_vm_pids or not mounted_by_zstack:
return
try:
kill_and_umount(mount_path, mount_path_is_nfs(mount_path))
except UmountException:
if shell.run('ps -p %s' % ' '.join(killed_vm_pids)) == 0:
virsh_list = shell.call(
"timeout 10 virsh list --all || echo 'cannot obtain virsh list'"
)
logger.debug("virsh_list:\n" + virsh_list)
logger.error(
'kill vm[pids:%s] failed because of unavailable fs[mountPath:%s].'
' please retry "umount -f %s"' %
(killed_vm_pids, mount_path, mount_path))
return
try_remount_fs()
def touch_heartbeat_file():
touch = shell.ShellCmd(
'timeout %s touch %s' %
(cmd.storageCheckerTimeout, heartbeat_file_path))
touch(False)
if touch.return_code != 0:
logger.warn(
'unable to touch %s, %s %s' %
(heartbeat_file_path, touch.stderr, touch.stdout))
return touch.return_code == 0
heartbeat_file_path = os.path.join(
mount_path, 'heartbeat-file-kvm-host-%s.hb' % cmd.hostUuid)
created_time = time.time()
with self.fencer_lock:
self.run_filesystem_fencer_timestamp[ps_uuid] = created_time
try:
failure = 0
url = shell.call("mount | grep -e '%s' | awk '{print $1}'" %
mount_path).strip()
options = shell.call(
"mount | grep -e '%s' | awk -F '[()]' '{print $2}'" %
mount_path).strip()
while self.run_filesystem_fencer(ps_uuid, created_time):
time.sleep(cmd.interval)
if touch_heartbeat_file():
failure = 0
continue
failure += 1
if failure == cmd.maxAttempts:
logger.warn(
'failed to touch the heartbeat file[%s] %s times, we lost the connection to the storage,'
'shutdown ourselves' %
(heartbeat_file_path, cmd.maxAttempts))
self.report_storage_status([ps_uuid], 'Disconnected')
killed_vm_pids = kill_vm(cmd.maxAttempts, [mount_path],
True)
after_kill_vm()
logger.debug('stop heartbeat[%s] for filesystem self-fencer' %
heartbeat_file_path)
except:
content = traceback.format_exc()
logger.warn(content)
for mount_path, uuid, mounted_by_zstack in zip(cmd.mountPaths,
cmd.uuids,
cmd.mountedByZStack):
if not linux.timeout_isdir(mount_path):
raise Exception('the mount path[%s] is not a directory' %
mount_path)
heartbeat_file_fencer(mount_path, uuid, mounted_by_zstack)
return jsonobject.dumps(AgentRsp())
def _get_file_size(self, path):
o = shell.call('rbd --format json info %s' % path)
o = jsonobject.loads(o)
return long(o.size_)
def callback(self, req):
rsp = jsonobject.loads(req[http.REQUEST_BODY])
print jsonobject.dumps(rsp)
def setup_ceph_self_fencer(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
mon_url = '\;'.join(cmd.monUrls)
mon_url = mon_url.replace(':', '\\\:')
self.run_ceph_fencer = True
def ceph_in_error_stat():
# HEALTH_OK,HEALTH_WARN,HEALTH_ERR and others...
healthStatus = shell.call('ceph health')
return not (healthStatus.startswith('HEALTH_OK')
or healthStatus.startswith('HEALTH_WARN'))
def heartbeat_file_exists():
touch = shell.ShellCmd(
'timeout %s qemu-img info rbd:%s:id=zstack:key=%s:auth_supported=cephx\;none:mon_host=%s'
% (cmd.storageCheckerTimeout, cmd.heartbeatImagePath,
cmd.userKey, mon_url))
touch(False)
if touch.return_code == 0:
return True
logger.warn('cannot query heartbeat image: %s: %s' %
(cmd.heartbeatImagePath, touch.stderr))
return False
def create_heartbeat_file():
create = shell.ShellCmd(
'timeout %s qemu-img create -f raw rbd:%s:id=zstack:key=%s:auth_supported=cephx\;none:mon_host=%s 1'
% (cmd.storageCheckerTimeout, cmd.heartbeatImagePath,
cmd.userKey, mon_url))
create(False)
if create.return_code == 0 or "File exists" in create.stderr:
return True
logger.warn('cannot create heartbeat image: %s: %s' %
(cmd.heartbeatImagePath, create.stderr))
return False
def delete_heartbeat_file():
delete = shell.ShellCmd(
"timeout %s rbd rm --id zstack %s -m %s" %
(cmd.storageCheckerTimeout, cmd.heartbeatImagePath, mon_url))
delete(False)
@thread.AsyncThread
def heartbeat_on_ceph():
try:
failure = 0
while self.run_ceph_fencer:
time.sleep(cmd.interval)
if heartbeat_file_exists() or create_heartbeat_file():
failure = 0
continue
failure += 1
if failure == cmd.maxAttempts:
# c.f. We discovered that, Ceph could behave the following:
# 1. Create heart-beat file, failed with 'File exists'
# 2. Query the hb file in step 1, and failed again with 'No such file or directory'
if ceph_in_error_stat():
path = (os.path.split(cmd.heartbeatImagePath))[0]
kill_vm(cmd.maxAttempts, path, False)
else:
delete_heartbeat_file()
# reset the failure count
failure = 0
logger.debug('stop self-fencer on ceph primary storage')
except:
content = traceback.format_exc()
logger.warn(content)
heartbeat_on_ceph()
return jsonobject.dumps(AgentRsp())
def rebase_root_volume_to_backing_file(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
linux.qcow2_rebase_no_check(cmd.backingFilePath, cmd.rootVolumePath)
return jsonobject.dumps(AgentResponse())
def cancel_filesystem_self_fencer(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
with self.fencer_lock:
for ps_uuid in cmd.psUuids:
self.run_filesystem_fencer_timestamp.pop(ps_uuid, None)
return jsonobject.dumps(AgentRsp())
def delete_target(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
self._delete_target(cmd.target, cmd.uuid)
logger.debug('deleted iscsi target[%s]' % cmd.target)
rsp = AgentCapacityResponse()
return jsonobject.dumps(rsp)
def download_image(self, req):
#TODO: report percentage to mgmt server
def percentage_callback(percent, url):
logger.debug('Downloading %s ... %s%%' % (url, percent))
def use_wget(url, name, workdir, timeout):
return linux.wget(url,
workdir=workdir,
rename=name,
timeout=timeout,
interval=2,
callback=percentage_callback,
callback_data=url)
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = DownloadResponse()
supported_schemes = [self.URL_HTTP, self.URL_HTTPS, self.URL_FILE]
if cmd.urlScheme not in supported_schemes:
rsp.success = False
rsp.error = 'unsupported url scheme[%s], SimpleSftpBackupStorage only supports %s' % (
cmd.urlScheme, supported_schemes)
return jsonobject.dumps(rsp)
path = os.path.dirname(cmd.installPath)
if not os.path.exists(path):
os.makedirs(path, 0777)
image_name = os.path.basename(cmd.installPath)
install_path = cmd.installPath
timeout = cmd.timeout if cmd.timeout else 7200
if cmd.urlScheme in [self.URL_HTTP, self.URL_HTTPS]:
try:
ret = use_wget(cmd.url, image_name, path, timeout)
if ret != 0:
rsp.success = False
rsp.error = 'http/https download failed, [wget -O %s %s] returns value %s' % (
image_name, cmd.url, ret)
return jsonobject.dumps(rsp)
except linux.LinuxError as e:
traceback.format_exc()
rsp.success = False
rsp.error = str(e)
return jsonobject.dumps(rsp)
elif cmd.urlScheme == self.URL_FILE:
src_path = cmd.url.lstrip('file:')
src_path = os.path.normpath(src_path)
if not os.path.isfile(src_path):
raise Exception('cannot find the file[%s]' % src_path)
shell.call('yes | cp %s %s' % (src_path, install_path))
os.chmod(cmd.installPath, stat.S_IRUSR + stat.S_IRGRP + stat.S_IROTH)
size = os.path.getsize(install_path)
image_format = bash_o(
"qemu-img info %s | grep -w '^file format' | awk '{print $3}'" %
install_path).strip('\n')
if "raw" in image_format:
if "ISO" in bash_o("file %s" % install_path):
image_format = "iso"
md5sum = 'not calculated'
logger.debug('successfully downloaded %s to %s' %
(cmd.url, install_path))
(total, avail) = self.get_capacity()
rsp.md5Sum = md5sum
rsp.actualSize = size
rsp.size = linux.qcow2_virtualsize(install_path)
rsp.totalCapacity = total
rsp.availableCapacity = avail
rsp.format = image_format
return jsonobject.dumps(rsp)
