def post_servers_1234_action(self, body, **kw):
assert_equal(len(body.keys()), 1)
action = body.keys()[0]
if action == 'reboot':
assert_equal(body[action].keys(), ['type'])
assert_in(body[action]['type'], ['HARD', 'SOFT'])
elif action == 'rebuild':
assert_equal(body[action].keys(), ['imageId'])
elif action == 'resize':
assert_equal(body[action].keys(), ['flavorId'])
elif action == 'createBackup':
assert_equal(set(body[action].keys()),
set(['name', 'rotation', 'backup_type']))
elif action == 'confirmResize':
assert_equal(body[action], None)
# This one method returns a different response code
return (204, None)
elif action == 'revertResize':
assert_equal(body[action], None)
elif action == 'migrate':
assert_equal(body[action], None)
elif action == 'addFixedIp':
assert_equal(body[action].keys(), ['networkId'])
elif action == 'removeFixedIp':
assert_equal(body[action].keys(), ['address'])
else:
fail("Unexpected server action: %s" % action)
return (202, None)
def main():
# Start service
try:
l = launchd.Launchd()
l.load(PLIST)
except OSError, e:
fail('Cannot load launchd job: {0}', e)
def gfid_to_path_using_batchfind(brick, gfids_file, output_file):
"""
find -samefile gets the inode number and crawls entire namespace
to get the list of files/dirs having same inode number.
Do find without any option, except the ignore directory option,
print the output in <INODE_NUM> <PATH> format, use this output
to look into in-memory dictionary of inode numbers got from the
list of GFIDs
"""
with open(output_file, "a+") as fout:
inode_dict = {}
with open(gfids_file) as f:
for gfid in f:
gfid = gfid.strip()
backend_path = os.path.join(brick, ".glusterfs",
gfid[0:2], gfid[2:4], gfid)
try:
inode_dict[str(os.stat(backend_path).st_ino)] = 1
except (IOError, OSError) as e:
if e.errno == ENOENT:
continue
else:
fail("%s Failed to convert to path from "
"GFID %s: %s" % (brick, gfid, e), logger=logger)
if not inode_dict:
return
def inode_filter(path):
try:
st = os.lstat(path)
except (OSError, IOError) as e:
if e.errno == ENOENT:
st = None
else:
raise
if st and inode_dict.get(str(st.st_ino), None):
return True
return False
brick_path_len = len(brick)
def output_callback(path):
path = path.strip()
path = path[brick_path_len+1:]
output_write(fout, path, args.output_prefix)
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
# Length of brick path, to remove from output path
find(brick, callback_func=output_callback,
filter_func=inode_filter,
ignore_dirs=ignore_dirs)
fout.flush()
os.fsync(fout.fileno())
def post_servers_1234_action(self, body, **kw):
assert_equal(len(body.keys()), 1)
action = body.keys()[0]
if action == "reboot":
assert_equal(body[action].keys(), ["type"])
assert_in(body[action]["type"], ["HARD", "SOFT"])
elif action == "rebuild":
assert_equal(body[action].keys(), ["imageId"])
elif action == "resize":
assert_equal(body[action].keys(), ["flavorId"])
elif action == "confirmResize":
assert_equal(body[action], None)
# This one method returns a different response code
return (204, None)
elif action == "revertResize":
assert_equal(body[action], None)
elif action == "migrate":
assert_equal(body[action], None)
elif action == "addFixedIp":
assert_equal(body[action].keys(), ["networkId"])
elif action == "removeFixedIp":
assert_equal(body[action].keys(), ["address"])
else:
fail("Unexpected server action: %s" % action)
return (202, None)
def mode_pre(session_dir, args):
global gtmpfilename
"""
Read from Session file and write to session.pre file
"""
endtime_to_update = int(time.time()) - get_changelog_rollover_time(
args.volume)
status_file = os.path.join(session_dir, args.volume, "status")
status_file_pre = status_file + ".pre"
mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
# If Pre status file exists and running pre command again
if os.path.exists(status_file_pre) and not args.regenerate_outfile:
fail("Post command is not run after last pre, "
"use --regenerate-outfile")
start = 0
try:
with open(status_file) as f:
start = int(f.read().strip())
except ValueError:
pass
except (OSError, IOError) as e:
fail("Error Opening Session file %s: %s" % (status_file, e),
logger=logger)
logger.debug("Pre is called - Session: %s, Volume: %s, "
"Start time: %s, End time: %s" %
(args.session, args.volume, start, endtime_to_update))
prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
gtmpfilename = prefix + next(tempfile._get_candidate_names())
run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename)
# Merger
if args.full:
cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
execute(cmd,
exit_msg="Failed to merge output files "
"collected from nodes",
logger=logger)
else:
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
write_output(args.outfile, outfilemerger, args.field_separator)
try:
os.remove(args.outfile + ".db")
except (IOError, OSError):
pass
run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename)
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(endtime_to_update))
sys.stdout.write("Generated output file %s\n" % args.outfile)
def validate_genesis(snapshot, genesis):
step('Validating genesis')
ddiff = DictDiffer(snapshot['genesis_state'], genesis)
added = ddiff.added()
removed = ddiff.removed()
changed = ddiff.changed()
if len(added) != 0 or len(removed) != 0 or len(changed) != 0:
fail()
if len(added) != 0:
print "> Snapshot has '%s' and your genesis dont" % (
','.join(added))
if len(removed) != 0:
print "> Your genesis has %s and snapshot dont" % (
','.join(removed))
if len(changed) != 0:
print "> Your genesis and snapshot have different '%s'" % (
','.join(changed))
print "# Snapshot genesis"
print json.dumps(snapshot['genesis_state'], indent=2, sort_keys=True)
print
print "# Your genesis"
print json.dumps(genesis, indent=2, sort_keys=True)
return False
success()
return True
def askTime(self, sumSecTime):
while True:
strTime = input(
"What's the time you want to show that picture at? (from 00:00 to 23:59, e.g. 22:30)\n"
)
try:
hours = int(strTime[0:2])
minutes = int(strTime[3:5])
seconds = (hours * 3600) + (minutes * 60)
if (0 <= hours <= 23 and 0 <= minutes <= 59
and strTime[2:3] == ":"):
if (self.isLater(sumSecTime, seconds)):
break
else:
utils.fail("NameError")
else:
#crash try() and force user to try again
utils.fail("ValueError")
except ValueError:
print("Invalid time format! Try again")
time.sleep(2)
except NameError:
print(
"This image must be set later than the last one! Try again"
)
time.sleep(2)
print()
return strTime
def main():
args = _get_args()
mkdirp(conf.get_opt("session_dir"), exit_on_err=True)
if args.mode == "list":
session_dir = conf.get_opt("session_dir")
else:
session_dir = os.path.join(conf.get_opt("session_dir"), args.session)
if not os.path.exists(session_dir) and args.mode not in ["create", "list"]:
fail("Invalid session %s" % args.session)
vol_dir = os.path.join(session_dir, args.volume)
if not os.path.exists(vol_dir) and args.mode not in ["create", "list"]:
fail("Session %s not created with volume %s" %
(args.session, args.volume))
mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume),
exit_on_err=True)
log_file = os.path.join(conf.get_opt("log_dir"), args.session, args.volume,
"cli.log")
setup_logger(logger, log_file, args.debug)
# globals() will have all the functions already defined.
# mode_<args.mode> will be the function name to be called
globals()["mode_" + args.mode](session_dir, args)
def info():
"""获取用户信息
---
tags:
- 用户
responses:
200:
description: 获取成功
schema:
type: object
properties:
code:
type: int
data:
type: array
$ref: '#/definitions/Module'
message:
type: string
examples:
code: 0
data: [{}, {}]
message: 'success'
"""
token = request.args.get('token')
data = verify_token(token)
if not data:
return fail(401)
user = Admin.query.get_or_404(data['user_id'])
if not user:
return fail(401)
res = {'data': {'name': user.name, 'avatar': user.avatar}}
return success(res)
def export(self, targz: str, **kwargs: dict):
if not os.path.isdir(self.path):
utils.fail("%s: VM does not exist" % self.name)
exit(1)
if not targz:
targz = '%s.tar.gz' % self.name
targz = os.path.abspath(targz)
utils.info("exporting to %s" % targz)
tar = '%s/%s' % (self.path, os.path.basename(os.path.splitext(targz)[0]))
if os.path.exists(targz):
utils.fail("%s: package already exists" % targz)
exit(1)
os.chdir(self.path) # create intermediate files in VM's path
utils.pend("package disk image")
utils.execute(['tar', '-cSf', tar, os.path.basename(self.path_raw)])
utils.ok()
for s in self.snapshots:
utils.pend("package snapshot: %s" % s)
local_snapshot = '%s.%s' % (os.path.basename(self.path_raw), s)
utils.execute(['tar', '-rf', tar, local_snapshot])
utils.ok()
utils.pend("compress package", msg="may take some time")
utils.execute(['gzip', '-c', tar], outfile=targz)
utils.ok()
utils.pend("clean up")
os.unlink(tar)
utils.ok()
self.scan_snapshots()
def exec(self, cmd, args='', expected_exit=0):
"""Execute binary in current test context"""
env = {**self.env, **os.environ.copy(), **self.test.utenv}
if sys.platform == 'win32':
env['PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('PATH', '')
cmd = os.path.join(self.build.exedir, cmd) + '.exe'
else:
if self.test.ld_preload:
env['LD_PRELOAD'] = env.get('LD_PRELOAD', '') + os.pathsep +\
self.test.ld_preload
self.valgrind.handle_ld_preload(self.test.ld_preload)
env['LD_LIBRARY_PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('LD_LIBRARY_PATH', '')
cmd = os.path.join(self.test.cwd, cmd) + self.build.exesuffix
cmd = '{} {}'.format(self.valgrind.cmd, cmd)
proc = sp.run('{} {}'.format(cmd, args), env=env, cwd=self.test.cwd,
shell=True, timeout=self.conf.timeout, stdout=sp.PIPE,
stderr=sp.STDOUT, universal_newlines=True)
if sys.platform != 'win32' and expected_exit == 0 \
and not self.valgrind.validate_log():
self.test.fail(proc.stdout)
if proc.returncode != expected_exit:
fail(proc.stdout, exit_code=proc.returncode)
else:
self.msg.print_verbose(proc.stdout)
def get_nodes(volume):
"""
Get the gluster volume info xml output and parse to get
the brick details.
"""
global vol_statusStr
cmd = ["gluster", 'volume', 'info', volume, "--xml"]
_, data, _ = execute(cmd,
exit_msg="Failed to Run Gluster Volume Info",
logger=logger)
tree = etree.fromstring(data)
# Test to check if volume has been deleted after session creation
count_el = tree.find('volInfo/volumes/count')
if int(count_el.text) == 0:
fail("Unable to get volume details", logger=logger)
# this status is used in caller: run_cmd_nodes
vol_statusStr = tree.find('volInfo/volumes/volume/statusStr').text
nodes = []
volume_el = tree.find('volInfo/volumes/volume')
try:
for b in volume_el.findall('bricks/brick'):
nodes.append((b.find('hostUuid').text, b.find('name').text))
except (ParseError, AttributeError, ValueError) as e:
fail("Failed to parse Volume Info: %s" % e, logger=logger)
return nodes
def exec(self, cmd, args='', expected_exit=0):
"""Execute binary in current test context"""
env = {**self.env, **os.environ.copy(), **self.test.utenv}
if sys.platform == 'win32':
env['PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('PATH', '')
cmd = os.path.join(self.build.exedir, cmd) + '.exe'
else:
if self.test.ld_preload:
env['LD_PRELOAD'] = env.get('LD_PRELOAD', '') + os.pathsep +\
self.test.ld_preload
self.valgrind.handle_ld_preload(self.test.ld_preload)
env['LD_LIBRARY_PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('LD_LIBRARY_PATH', '')
cmd = os.path.join(self.test.cwd, cmd) + self.build.exesuffix
cmd = '{} {}'.format(self.valgrind.cmd, cmd)
proc = sp.run('{} {}'.format(cmd, args), env=env, cwd=self.test.cwd,
shell=True, timeout=self.conf.timeout, stdout=sp.PIPE,
stderr=sp.STDOUT, universal_newlines=True)
if sys.platform != 'win32' and expected_exit == 0 \
and not self.valgrind.validate_log():
self.test.fail(proc.stdout)
if proc.returncode != expected_exit:
fail(proc.stdout, exit_code=proc.returncode)
else:
self.msg.print_verbose(proc.stdout)
def mode_create(session_dir, args):
validate_session_name(args.session)
logger.debug("Init is called - Session: %s, Volume: %s" %
(args.session, args.volume))
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume),
exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
enable_volume_options(args)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w") as f:
f.write(str(time_to_update))
sys.stdout.write("Session %s created with volume %s\n" %
(args.session, args.volume))
sys.exit(0)
def _run_match(self):
"""Match log files"""
cwd_listdir = [path.join(self.cwd, f) for f in listdir(self.cwd)]
suffix = '{}.log.match'.format(self.testnum)
def is_matchfile(f):
"""Match file ends with specific suffix and a char before suffix
is not a digit"""
before_suffix = -len(suffix) - 1
return path.isfile(f) and f.endswith(suffix) and \
not f[before_suffix].isdigit()
match_files = filter(is_matchfile, cwd_listdir)
prefix = 'perl ' if sys.platform == 'win32' else ''
match_cmd = prefix + path.join(hlp.ROOTDIR, 'match')
for mf in match_files:
cmd = '{} {}'.format(match_cmd, mf)
proc = sp.run(cmd.split(), stdout=sp.PIPE, cwd=self.cwd,
stderr=sp.STDOUT, universal_newlines=True)
if proc.returncode != 0:
fail(proc.stdout, exit_code=proc.returncode)
else:
self.msg.print_verbose(proc.stdout)
def mode_pre(session_dir, args):
global gtmpfilename
"""
Read from Session file and write to session.pre file
"""
endtime_to_update = int(time.time()) - get_changelog_rollover_time(
args.volume)
status_file = os.path.join(session_dir, args.volume, "status")
status_file_pre = status_file + ".pre"
mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
# If Pre status file exists and running pre command again
if os.path.exists(status_file_pre) and not args.regenerate_outfile:
fail("Post command is not run after last pre, "
"use --regenerate-outfile")
start = 0
try:
with open(status_file) as f:
start = int(f.read().strip())
except ValueError:
pass
except (OSError, IOError) as e:
fail("Error Opening Session file %s: %s"
% (status_file, e), logger=logger)
logger.debug("Pre is called - Session: %s, Volume: %s, "
"Start time: %s, End time: %s"
% (args.session, args.volume, start, endtime_to_update))
prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
gtmpfilename = prefix + next(tempfile._get_candidate_names())
run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename)
# Merger
if args.full:
cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
execute(cmd,
exit_msg="Failed to merge output files "
"collected from nodes", logger=logger)
else:
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
write_output(args.outfile, outfilemerger, args.field_separator)
try:
os.remove(args.outfile + ".db")
except (IOError, OSError):
pass
run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename)
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(endtime_to_update))
sys.stdout.write("Generated output file %s\n" % args.outfile)
def main(test_filter=None):
if not test_filter:
test_filter = "*"
try:
test_suite = list(TestParser.read_all(test_filter))
except (UnknownFieldError, RequiredFieldError) as error:
print(fail(str(error)))
exit(1)
total_tests = 0
for fname, suite_tests in test_suite:
total_tests += len(suite_tests)
failed_tests = []
print(
ok("[==========]") + " Running %d tests from %d test cases.\n" %
(total_tests, len(test_suite)))
start_time = time.time()
skipped_tests = []
for fname, tests in test_suite:
print(ok("[----------]") + " %d tests from %s" % (len(tests), fname))
for test in tests:
status = Utils.run_test(test)
if Utils.skipped(status):
skipped_tests.append((fname, test, status))
if Utils.failed(status):
failed_tests.append("%s.%s" % (fname, test.name))
# TODO(mmarchini) elapsed time per test suite and per test (like gtest)
print(ok("[----------]") + " %d tests from %s\n" % (len(tests), fname))
elapsed = time.time() - start_time
total_tests -= len(skipped_tests)
# TODO(mmarchini) pretty print time
print(
ok("[==========]") + " %d tests from %d test cases ran. (%s total)" %
(total_tests, len(test_suite), elapsed))
print(
ok("[ PASSED ]") + " %d tests." % (total_tests - len(failed_tests)))
if skipped_tests:
print(
warn("[ SKIP ]") +
" %d tests, listed below:" % len(skipped_tests))
for test_suite, test, status in skipped_tests:
print(
warn("[ SKIP ]") + " %s.%s (%s)" %
(test_suite, test.name, Utils.skip_reason(test, status)))
if failed_tests:
print(
fail("[ FAILED ]") +
" %d tests, listed below:" % len(failed_tests))
for failed_test in failed_tests:
print(fail("[ FAILED ]") + " %s" % failed_test)
if failed_tests:
exit(1)
async def upimg(request):
# 判断用户是否具有上传权限
if request.headers.get('token') != TOKEN:
return fail('您没有使用本服务的权限')
# 判断参数是否正确
if not request.files and not request.files.get('file'):
return fail('error args')
image = request.files.get('file').body
# 判断文件是否支持
imageSuffix = getSuffix(bytes2hex(image))
if 'error' in imageSuffix:
return fail(imageSuffix)
# 组织图片存储路径
m1 = hashlib.md5()
m1.update(image)
md5Name = m1.hexdigest()
saveDir = baseDir + md5Name[0:2] + '/'
savePath = saveDir + md5Name[2:] + '.' + imageSuffix
resPath = '/' + md5Name[0:2] + '/' + md5Name[2:] + '.' + imageSuffix
# 如果文件夹不存在，就创建文件夹
if not os.path.exists(saveDir):
os.makedirs(saveDir)
# 将文件写入到硬盘
tempFile = open(savePath, 'wb')
tempFile.write(image)
tempFile.close()
# 给客户端返回结果
return ok({"path": resPath})
def _run_match(self):
"""Match log files"""
cwd_listdir = [path.join(self.cwd, f) for f in os.listdir(self.cwd)]
suffix = '{}.log.match'.format(self.testnum)
def is_matchfile(f):
"""Match file ends with specific suffix and a char before suffix
is not a digit"""
before_suffix = -len(suffix) - 1
return path.isfile(f) and f.endswith(suffix) and \
not f[before_suffix].isdigit()
match_files = filter(is_matchfile, cwd_listdir)
prefix = 'perl ' if sys.platform == 'win32' else ''
match_cmd = prefix + path.join(hlp.ROOTDIR, 'match')
for mf in match_files:
cmd = '{} {}'.format(match_cmd, mf)
proc = sp.run(cmd.split(), stdout=sp.PIPE, cwd=self.cwd,
stderr=sp.STDOUT, universal_newlines=True)
if proc.returncode != 0:
fail(proc.stdout, exit_code=proc.returncode)
else:
self.msg.print_verbose(proc.stdout)
def mode_pre(session_dir, args):
"""
Read from Session file and write to session.pre file
"""
endtime_to_update = int(time.time()) - int(
conf.get_opt("changelog_rollover_time"))
status_file = os.path.join(session_dir, args.volume, "status")
status_file_pre = status_file + ".pre"
mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
start = 0
try:
with open(status_file) as f:
start = int(f.read().strip())
except ValueError:
pass
except (OSError, IOError) as e:
fail("Error Opening Session file %s: %s"
% (status_file, e), logger=logger)
logger.debug("Pre is called - Session: %s, Volume: %s, "
"Start time: %s, End time: %s"
% (args.session, args.volume, start, endtime_to_update))
run_in_nodes(args.volume, start, args)
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(endtime_to_update))
sys.stdout.write("Generated output file %s\n" % args.outfile)
def get_nodes(volume):
"""
Get the gluster volume info xml output and parse to get
the brick details.
"""
global vol_statusStr;
cmd = ["gluster", 'volume', 'info', volume, "--xml"]
_, data, _ = execute(cmd,
exit_msg="Failed to Run Gluster Volume Info",
logger=logger)
tree = etree.fromstring(data)
# Test to check if volume has been deleted after session creation
count_el = tree.find('volInfo/volumes/count')
if int(count_el.text) == 0:
fail("Unable to get volume details", logger=logger)
# this status is used in caller: run_cmd_nodes
vol_statusStr = tree.find('volInfo/volumes/volume/statusStr').text
nodes = []
volume_el = tree.find('volInfo/volumes/volume')
try:
for b in volume_el.findall('bricks/brick'):
nodes.append((b.find('hostUuid').text,
b.find('name').text))
except (ParseError, AttributeError, ValueError) as e:
fail("Failed to parse Volume Info: %s" % e, logger=logger)
return nodes
def parse_acquisitions(self):
src_paths = []
for f in self._files:
# Check if it is a DICOM
try:
ds = dicom.read_file(f)
except InvalidDicomError:
utils.warning('Not a DICOM file: {}'.format(f))
continue
# Filter by serie description
dsDesc = ds.SeriesDescription
if any(_ in dsDesc for _ in self.exclusionCriteria):
self._excluded.add(dsDesc)
else:
src_paths.append(f)
groups = dcmstack.parse_and_group(src_paths)
for key, group in groups.iteritems():
if self._isFromOneDirectory(group):
#utils.ok("Same directory: {}".format(key[2]))
stack = dcmstack.stack_group(group)
inDir = os.path.dirname(group[0][2])
self._acquisitions.append(
Acquisition(inDir, stack, self._session))
else:
#TODO: regroup dicoms in a tmp directory
utils.new_line()
utils.fail(
"DICOM of '{}' are not in the same directory.\nThis structure won't be compatible with dcm2niibatch"
.format(key[2]))
def main():
args = _get_args()
mkdirp(conf.get_opt("session_dir"), exit_on_err=True)
if args.mode == "list":
session_dir = conf.get_opt("session_dir")
else:
session_dir = os.path.join(conf.get_opt("session_dir"),
args.session)
if not os.path.exists(session_dir) and args.mode not in ["create", "list"]:
fail("Invalid session %s" % args.session)
vol_dir = os.path.join(session_dir, args.volume)
if not os.path.exists(vol_dir) and args.mode not in ["create", "list"]:
fail("Session %s not created with volume %s" %
(args.session, args.volume))
mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume),
exit_on_err=True)
log_file = os.path.join(conf.get_opt("log_dir"),
args.session,
args.volume,
"cli.log")
setup_logger(logger, log_file, args.debug)
# globals() will have all the functions already defined.
# mode_<args.mode> will be the function name to be called
globals()["mode_" + args.mode](session_dir, args)
def mode_create(session_dir, args):
validate_session_name(args.session)
logger.debug("Init is called - Session: %s, Volume: %s"
% (args.session, args.volume))
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
enable_volume_options(args)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w") as f:
f.write(str(time_to_update))
sys.stdout.write("Session %s created with volume %s\n" %
(args.session, args.volume))
sys.exit(0)
def read(self, arg):
newresult = {}
f = open(self.arg_id(arg), "r")
for line in f.readlines():
line = line.strip().split()
if len(line) % 2 == 1 or len(line) < 2:
# Fail
utils.fail(self.arg_id(arg))
f.close()
return
new_key = {k: v for k, v in arg}
# parse line
for i in xrange(0, len(line) - 2, 2):
i_name = line[i]
i_value = float(line[i + 1])
new_key[i_name] = i_value
new_key["name"] = line[-2]
newresult[utils.dict2tuple(new_key)] = float(line[-1])
# Merge result with newresult
for k in newresult.keys():
self.result[k] = newresult[k]
f.close()
def main():
# Bind to service port
server_port = mach.BootstrapServer.lookup("org.freenas.test.mach.ipc-server")
local_port = mach.Port()
print 'Service port: {0}'.format(server_port)
print 'Local port: {0}'.format(local_port)
# Send a few messages
for i in range(0, 100):
msg = mach.Message()
msg.bits = mach.make_msg_bits(
mach.MessageType.MACH_MSG_TYPE_COPY_SEND,
mach.MessageType.MACH_MSG_TYPE_MAKE_SEND
)
msg.body = bytearray(random_str())
local_port.send(server_port, msg)
reply = local_port.receive()
print 'Received reply: {0}'.format(reply.body)
if reply.body != msg.body:
fail('Reply mismatch: {0} != {1}'.format(msg.body, reply.body))
# Exit
msg = mach.Message()
msg.bits = mach.make_msg_bits(
mach.MessageType.MACH_MSG_TYPE_COPY_SEND,
mach.MessageType.MACH_MSG_TYPE_MAKE_SEND
)
msg.body = bytearray('EXIT')
mach.null_port.send(server_port, msg)
def mode_pre(session_dir, args):
"""
Read from Session file and write to session.pre file
"""
endtime_to_update = int(time.time()) - int(
conf.get_opt("changelog_rollover_time"))
status_file = os.path.join(session_dir, args.volume, "status")
status_file_pre = status_file + ".pre"
mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
start = 0
try:
with open(status_file) as f:
start = int(f.read().strip())
except ValueError:
pass
except (OSError, IOError) as e:
fail("Error Opening Session file %s: %s" % (status_file, e),
logger=logger)
logger.debug("Pre is called - Session: %s, Volume: %s, "
"Start time: %s, End time: %s" %
(args.session, args.volume, start, endtime_to_update))
run_in_nodes(args.volume, start, args)
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(endtime_to_update))
sys.stdout.write("Generated output file %s\n" % args.outfile)
def main():
# Create send port
try:
send = mach.Port()
send.insert_right(mach.MessageType.MACH_MSG_TYPE_MAKE_SEND)
print 'Send port: {0}'.format(send)
except mach.MachException, e:
fail('Cannot create send port: {0}'.format(e))
def main():
# Create send port
try:
send = mach.Port()
send.insert_right(mach.MessageType.MACH_MSG_TYPE_MAKE_SEND)
print 'Send port: {0}'.format(send)
except mach.MachException, e:
fail('Cannot create send port: {0}'.format(e))
def vm_init(path: str):
utils.pend("initialise VM directory: %s" % path)
try:
os.mkdir(path)
except OSError as e:
utils.fail(e.strip())
exit(1)
utils.ok()
def mode_list(session_dir, args):
"""
List available sessions to stdout, if session name is set
only list that session.
"""
if args.session:
if not os.path.exists(os.path.join(session_dir, args.session)):
fail("Invalid Session", logger=logger)
sessions = [args.session]
else:
sessions = []
for d in os.listdir(session_dir):
if d != ".keys":
sessions.append(d)
output = []
for session in sessions:
# Session Volume Last Processed
volnames = os.listdir(os.path.join(session_dir, session))
for volname in volnames:
if args.volume and args.volume != volname:
continue
status_file = os.path.join(session_dir, session, volname, "status")
last_processed = None
try:
with open(status_file) as f:
last_processed = f.read().strip()
except (OSError, IOError) as e:
if e.errno == ENOENT:
continue
else:
raise
output.append((session, volname, last_processed))
if output:
sys.stdout.write("%s %s %s\n" %
("SESSION".ljust(25), "VOLUME".ljust(25),
"SESSION TIME".ljust(25)))
sys.stdout.write("-" * 75)
sys.stdout.write("\n")
for session, volname, last_processed in output:
sess_time = 'Session Corrupted'
if last_processed:
try:
sess_time = human_time(last_processed)
except TypeError:
sess_time = 'Session Corrupted'
sys.stdout.write(
"%s %s %s\n" %
(session.ljust(25), volname.ljust(25), sess_time.ljust(25)))
if not output:
if args.session or args.volume:
fail("Invalid Session", logger=logger)
else:
sys.stdout.write("No sessions found.\n")
def mode_list(session_dir, args):
"""
List available sessions to stdout, if session name is set
only list that session.
"""
if args.session:
if not os.path.exists(os.path.join(session_dir, args.session)):
fail("Invalid Session", logger=logger)
sessions = [args.session]
else:
sessions = []
for d in os.listdir(session_dir):
if d != ".keys":
sessions.append(d)
output = []
for session in sessions:
# Session Volume Last Processed
volnames = os.listdir(os.path.join(session_dir, session))
for volname in volnames:
if args.volume and args.volume != volname:
continue
status_file = os.path.join(session_dir, session, volname, "status")
last_processed = None
try:
with open(status_file) as f:
last_processed = f.read().strip()
except (OSError, IOError) as e:
if e.errno == ENOENT:
pass
else:
raise
output.append((session, volname, last_processed))
if output:
sys.stdout.write("%s %s %s\n" % ("SESSION".ljust(25),
"VOLUME".ljust(25),
"SESSION TIME".ljust(25)))
sys.stdout.write("-"*75)
sys.stdout.write("\n")
for session, volname, last_processed in output:
sess_time = 'Session Corrupted'
if last_processed:
try:
sess_time = human_time(last_processed)
except TypeError:
sess_time = 'Session Corrupted'
sys.stdout.write("%s %s %s\n" % (session.ljust(25),
volname.ljust(25),
sess_time.ljust(25)))
if not output:
if args.session or args.volume:
fail("Invalid Session", logger=logger)
else:
sys.stdout.write("No sessions found.\n")
def mode_create(session_dir, args):
logger.debug("Init is called - Session: %s, Volume: %s" %
(args.session, args.volume))
execute(["gluster", "volume", "info", args.volume],
exit_msg="Unable to get volume details",
logger=logger)
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume),
exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
execute(["gluster", "volume", "set", args.volume, "build-pgfid", "on"],
exit_msg="Failed to set volume option build-pgfid on",
logger=logger)
logger.info("Volume option set %s, build-pgfid on" % args.volume)
execute([
"gluster", "volume", "set", args.volume, "changelog.changelog",
"on"
],
exit_msg="Failed to set volume option "
"changelog.changelog on",
logger=logger)
logger.info("Volume option set %s, changelog.changelog on" %
args.volume)
execute([
"gluster", "volume", "set", args.volume,
"changelog.capture-del-path", "on"
],
exit_msg="Failed to set volume option "
"changelog.capture-del-path on",
logger=logger)
logger.info("Volume option set %s, changelog.capture-del-path on" %
args.volume)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w", buffering=0) as f:
f.write(str(time_to_update))
sys.exit(0)
def node_cmd(host, host_uuid, task, cmd, args, opts):
"""
Runs command via ssh if host is not local
"""
try:
localdir = is_host_local(host_uuid)
# this is so to avoid deleting the ssh keys on local node which
# otherwise cause ssh password prompts on the console (race conditions)
# mode_delete() should be cleaning up the session tree
if localdir and task == "delete":
return
pem_key_path = get_pem_key_path(args.session, args.volume)
if not localdir:
# prefix with ssh command if not local node
cmd = [
"ssh",
"-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no",
# We force TTY allocation (-t -t) so that Ctrl+C is handed
# through; see:
# https://bugzilla.redhat.com/show_bug.cgi?id=1382236
# Note that this turns stderr of the remote `cmd`
# into stdout locally.
"-t",
"-t",
"-i",
pem_key_path,
"root@%s" % host
] + cmd
(returncode, err, out) = execute(cmd, logger=logger)
if returncode != 0:
# Because the `-t -t` above turns the remote stderr into
# local stdout, we need to log both stderr and stdout
# here to print all error messages.
fail("%s - %s failed; stdout (including remote stderr):\n"
"%s\n"
"stderr:\n"
"%s" % (host, task, out, err),
returncode,
logger=logger)
if opts.get("copy_outfile", False) and not localdir:
cmd_copy = [
"scp", "-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no", "-i", pem_key_path,
"root@%s:/%s" % (host, opts.get("node_outfile")),
os.path.dirname(opts.get("node_outfile"))
]
execute(cmd_copy,
exit_msg="%s - Copy command failed" % host,
logger=logger)
except KeyboardInterrupt:
sys.exit(2)
def main() -> None:
for func in utils.get_functions():
if not func.decomp_name:
continue
if func.status == utils.FunctionStatus.Matching:
if not check_function(func.addr, func.size, func.decomp_name):
utils.fail(f"{func.decomp_name} was marked as matching but does not match")
return
def open_output(self):
try:
output_file = self.output + ("/out-%04i.txt" % self.number)
self.fd = open(output_file, "w")
except Exception as e:
print("")
from utils import fail
fail("Could not open task output file: " + output_file + "\n" +
str(e))
def dump_fn(name: str) -> None:
expected_dir = root / "expected"
try:
fn = get_fn_from_my_elf(name)
path = expected_dir / f"{name}.bin"
path.parent.mkdir(exist_ok=True)
path.write_bytes(fn)
except KeyError:
utils.fail("could not find function")
def set_parts(self, *parts):
"""
Adds a main pool to the poolset file.
The function accepts a list of Parts of any length.
Should not be called more than once.
"""
if not self.parts:
self.parts = list(parts)
else:
fail('This function should not be called more than once.')
def import_raw(self, raw: str, force: bool):
if not os.path.exists(raw):
utils.fail("%s: file not found" % raw)
exit(1)
self.initialise(force)
utils.pend("copy disk image")
utils.execute(['cp', raw, self.path_raw])
utils.ok()
def set_parts(self, *parts):
"""
Adds a main pool to the poolset file.
The function accepts a list of Parts of any length.
Should not be called more than once.
"""
if not self.parts:
self.parts = list(parts)
else:
fail('This function should not be called more than once.')
def setValues(self, root: Node):
if (len(root.childs) == 0):
if (root.info in self.eval_config):
root.truth = self.eval_config[root.info]
else:
fail("404", (root.info, ))
else:
for child in root.childs:
self.setValues(child)
self.was_set = True
def __init__(self, path_yaml: str):
if not os.path.isfile(path_yaml):
utils.fail("%s: file not found" % path_yaml)
exit(1)
yaml = Batch.YAML(path_yaml)
self.variables = yaml.tree['variables']
self.commands = []
for ctoken, i in zip(yaml.tree['commands'], range(len(yaml.tree['commands']))):
c = Batch.Command(ctoken, self.variables)
self.commands.append(c)
def validate_volume(volume):
cmd = ["gluster", 'volume', 'info', volume, "--xml"]
_, data, _ = execute(cmd,
exit_msg="Failed to Run Gluster Volume Info",
logger=logger)
try:
tree = etree.fromstring(data)
statusStr = tree.find('volInfo/volumes/volume/statusStr').text
except (ParseError, AttributeError) as e:
fail("Invalid Volume: Check the Volume name! %s" % e)
if statusStr != "Started":
fail("Volume %s is not online" % volume)
def update_evaluator(self, config: dict):
for k in config:
v = config[k]
if (v == "True"):
config[k] = True
elif (v == "False"):
config[k] = False
else:
from utils import fail
fail("config", (k, v))
self.evaluator.eval_config = config
def node_cmd(host, host_uuid, task, cmd, args, opts):
"""
Runs command via ssh if host is not local
"""
try:
localdir = is_host_local(host_uuid)
# this is so to avoid deleting the ssh keys on local node which
# otherwise cause ssh password prompts on the console (race conditions)
# mode_delete() should be cleaning up the session tree
if localdir and task == "delete":
return
pem_key_path = get_pem_key_path(args.session, args.volume)
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh",
"-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no",
# We force TTY allocation (-t -t) so that Ctrl+C is handed
# through; see:
# https://bugzilla.redhat.com/show_bug.cgi?id=1382236
# Note that this turns stderr of the remote `cmd`
# into stdout locally.
"-t",
"-t",
"-i", pem_key_path,
"root@%s" % host] + cmd
(returncode, err, out) = execute(cmd, logger=logger)
if returncode != 0:
# Because the `-t -t` above turns the remote stderr into
# local stdout, we need to log both stderr and stdout
# here to print all error messages.
fail("%s - %s failed; stdout (including remote stderr):\n"
"%s\n"
"stderr:\n"
"%s" % (host, task, out, err),
returncode,
logger=logger)
if opts.get("copy_outfile", False) and not localdir:
cmd_copy = ["scp",
"-oNumberOfPasswordPrompts=0",
"-oStrictHostKeyChecking=no",
"-i", pem_key_path,
"root@%s:/%s" % (host, opts.get("node_outfile")),
os.path.dirname(opts.get("node_outfile"))]
execute(cmd_copy, exit_msg="%s - Copy command failed" % host,
logger=logger)
except KeyboardInterrupt:
sys.exit(2)
def check_depth(self, root: Node):
"""Check the depth, will call the recursive one
Arguments:
root {Node} -- The starting node
"""
if (self.expected_depth[0] == 0):
self.get_expected_depth(self.input_phrase)
ch = self.check_depth_rec(root, 1)
if (ch != self.expected_depth[0] or 0 != self.expected_depth[1]
or root.childs[0].info == ""):
fail("ret", (-1, ))
def validate_volume(volume):
cmd = ["gluster", 'volume', 'info', volume, "--xml"]
_, data, _ = execute(cmd,
exit_msg="Failed to Run Gluster Volume Info",
logger=logger)
try:
tree = etree.fromstring(data)
statusStr = tree.find('volInfo/volumes/volume/statusStr').text
except (ParseError, AttributeError) as e:
fail("Invalid Volume: Check the Volume name! %s" % e)
if statusStr != "Started":
fail("Volume %s is not online" % volume)
def execute(args, cmd_line):
# Informative:
ip = utils.get_default_ip()
if args['headless']:
utils.info("VNC: port %d (connect with `$vncclient %s:%d`)"
% (args['vnc_port'], ip, args['vnc_display']))
utils.info("Qemu monitor: port %d (connect with `{nc,telnet} %s %d)"
% (args['monitor_port'], ip, args['monitor_port']))
if args['mode'] == 'sym':
utils.info("Experiment name: %s" % args['expname'])
if args['script'] or args['command']:
utils.info("Watchdog: port %d" % args['command_port'])
utils.debug("Command line:\n%s" % ' '.join(cmd_line))
if args['dry_run']:
exit(1)
environ = dict(os.environ)
if args['mode'] == 'sym':
environ['LUA_PATH'] = ';'.join(['%s/?.lua' % args['config_root'],
environ.get('LUA_PATH', '')])
utils.debug("LUA_PATH=%s" % environ['LUA_PATH'])
# each experiment gets its own directory:
try:
utils.pend("Creating experiment directory %s" % args['exppath'])
os.makedirs(args['exppath'])
utils.ok()
except FileExistsError:
utils.fail("Experiment %s already exists. Please choose another name."
% args['expname'])
exit(1)
if args['timeout']:
kill_me_later(args['timeout'])
obj = None
if args['script']:
module_file, test = args['script']
with open(module_file, 'r') as f:
code = f.read()
obj = Script(code=code, test=test)
if args['command']:
obj = Command.from_cmd_args(args['command'], args['env_var'] or [])
if obj:
async_send_command(obj, 'localhost', args['command_port'], TIMEOUT_CMD)
# drop `s2e-last` symlink somewhere where it does not get in the way:
os.chdir(utils.CHEFROOT_EXPDATA)
os.execvpe(cmd_line[0], cmd_line, environ)
def mode_create(session_dir, args):
logger.debug("Init is called - Session: %s, Volume: %s"
% (args.session, args.volume))
execute(["gluster", "volume", "info", args.volume],
exit_msg="Unable to get volume details",
logger=logger)
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
execute(["gluster", "volume", "set",
args.volume, "build-pgfid", "on"],
exit_msg="Failed to set volume option build-pgfid on",
logger=logger)
logger.info("Volume option set %s, build-pgfid on" % args.volume)
execute(["gluster", "volume", "set",
args.volume, "changelog.changelog", "on"],
exit_msg="Failed to set volume option "
"changelog.changelog on", logger=logger)
logger.info("Volume option set %s, changelog.changelog on"
% args.volume)
execute(["gluster", "volume", "set",
args.volume, "changelog.capture-del-path", "on"],
exit_msg="Failed to set volume option "
"changelog.capture-del-path on", logger=logger)
logger.info("Volume option set %s, changelog.capture-del-path on"
% args.volume)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w", buffering=0) as f:
f.write(str(time_to_update))
sys.exit(0)
def ssh_setup(args):
pem_key_path = get_pem_key_path(args.session, args.volume)
if not os.path.exists(pem_key_path):
# Generate ssh-key
cmd = ["ssh-keygen",
"-N",
"",
"-f",
pem_key_path]
execute(cmd,
exit_msg="Unable to generate ssh key %s"
% pem_key_path,
logger=logger)
logger.info("Ssh key generated %s" % pem_key_path)
try:
shutil.copyfile(pem_key_path + ".pub",
os.path.join(conf.get_opt("session_dir"),
".keys",
"%s_%s_secret.pem.pub" % (args.session,
args.volume)))
except (IOError, OSError) as e:
fail("Failed to copy public key to %s: %s"
% (os.path.join(conf.get_opt("session_dir"), ".keys"), e),
logger=logger)
# Copy pub file to all nodes
cmd = ["gluster",
"system::",
"copy",
"file",
"/glusterfind/.keys/%s.pub" % os.path.basename(pem_key_path)]
execute(cmd, exit_msg="Failed to distribute ssh keys", logger=logger)
logger.info("Distributed ssh key to all nodes of Volume")
# Add to authorized_keys file in each node
cmd = ["gluster",
"system::",
"execute",
"add_secret_pub",
"root",
"/glusterfind/.keys/%s.pub" % os.path.basename(pem_key_path)]
execute(cmd,
exit_msg="Failed to add ssh keys to authorized_keys file",
logger=logger)
logger.info("Ssh key added to authorized_keys of Volume nodes")
def ssh_setup(args):
pem_key_path = get_pem_key_path(args.session, args.volume)
if not os.path.exists(pem_key_path):
# Generate ssh-key
cmd = ["ssh-keygen",
"-N",
"",
"-f",
pem_key_path]
execute(cmd,
exit_msg="Unable to generate ssh key %s"
% pem_key_path,
logger=logger)
logger.info("Ssh key generated %s" % pem_key_path)
try:
shutil.copyfile(pem_key_path + ".pub",
os.path.join(conf.get_opt("session_dir"),
".keys",
"%s_%s_secret.pem.pub" % (args.session,
args.volume)))
except (IOError, OSError) as e:
fail("Failed to copy public key to %s: %s"
% (os.path.join(conf.get_opt("session_dir"), ".keys"), e),
logger=logger)
# Copy pub file to all nodes
cmd = ["gluster",
"system::",
"copy",
"file",
"/glusterfind/.keys/%s.pub" % os.path.basename(pem_key_path)]
execute(cmd, exit_msg="Failed to distribute ssh keys", logger=logger)
logger.info("Distributed ssh key to all nodes of Volume")
# Add to authorized_keys file in each node
cmd = ["gluster",
"system::",
"execute",
"add_secret_pub",
"root",
"/glusterfind/.keys/%s.pub" % os.path.basename(pem_key_path)]
execute(cmd,
exit_msg="Failed to add ssh keys to authorized_keys file",
logger=logger)
logger.info("Ssh key added to authorized_keys of Volume nodes")
def check(self, root: Node):
if (root.info == ""):
return False
if len(root.childs) > 2:
fail("syntax", (root, ))
return False
for child in root.childs:
if (self.check_childs(child)):
self.check(child)
else:
fail("syntax", (child, ))
return False
return True
def node_run(volume, host, path, start, outfile, args, fallback=False):
"""
If host is local node, execute the command locally. If not local
execute the CHANGE_DETECTOR command via ssh and copy the output file from
remote node using scp.
"""
localdir = is_host_local(host)
pem_key_path = get_pem_key_path(args.session, args.volume)
# If Full backup is requested or start time is zero, use brickfind
change_detector = conf.get_change_detector(args.change_detector)
if ((start == 0 or args.full) and args.change_detector == "changelog") or \
fallback:
change_detector = conf.get_change_detector("brickfind")
# CHANGE_DETECTOR <SESSION> <VOLUME> <BRICK> <OUTFILE> <START> --debug
# --gfidpath <TYPE>
cmd = [change_detector,
args.session,
volume,
path,
outfile,
str(start),
"--output-prefix",
args.output_prefix] + \
(["--debug"] if args.debug else []) + \
(["--full"] if args.full else [])
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh",
"-i", pem_key_path,
"root@%s" % host] + cmd
rc, out, err = execute(cmd, logger=logger)
if rc == 2:
# Partial History Fallback
logger.info("%s %s Fallback to brickfind" % (host, err.strip()))
# Exit only from process, handled in main.
sys.exit(rc)
elif rc != 0:
fail("%s - Change detection failed" % host, logger=logger)
if not localdir:
cmd_copy = ["scp",
"-i", pem_key_path,
"root@%s:/%s" % (host, outfile),
os.path.dirname(outfile)]
execute(cmd_copy, exit_msg="%s - Copy command failed" % host,
logger=logger)
def delete(self, snapshot: str=None, **kwargs: dict):
if snapshot:
utils.pend("delete snapshot %s" % kwargs['name[:snapshot]'])
if snapshot not in self.snapshots:
utils.fail("%s: snapshot does not exist" % kwargs['name[:snapshot]'])
exit(1)
try:
os.unlink('%s.%s' % (self.path_raw, snapshot))
utils.ok()
except PermissionError:
utils.fail("Permission denied")
exit(1)
else:
if not os.path.isdir(self.path):
utils.fail("%s: VM does not exist" % self.name)
exit(1)
if not utils.ask("Delete VM %s?" % self.name, default=False):
exit(1)
utils.pend("delete %s" % self.name)
try:
shutil.rmtree(self.path)
utils.ok()
except PermissionError:
utils.fail("Permission denied")
exit(1)
def node_run(volume, host, path, start, outfile, args, fallback=False):
"""
If host is local node, execute the command locally. If not local
execute the CHANGE_DETECTOR command via ssh and copy the output file from
remote node using scp.
"""
localdir = is_host_local(host)
# If Full backup is requested or start time is zero, use brickfind
change_detector = conf.get_change_detector(args.change_detector)
if ((start == 0 or args.full) and args.change_detector == "changelog") or \
fallback:
change_detector = conf.get_change_detector("brickfind")
# CHANGE_DETECTOR <SESSION> <VOLUME> <BRICK> <OUTFILE> <START> --debug
# --gfidpath <TYPE>
cmd = [change_detector,
args.session,
volume,
path,
outfile,
str(start),
"--output-prefix",
args.output_prefix] + \
(["--debug"] if args.debug else []) + \
(["--full"] if args.full else [])
if not localdir:
# prefix with ssh command if not local node
cmd = ["ssh", "-i", conf.get_opt("secret_pem"), "root@%s" % host] + cmd
rc, out, err = execute(cmd, logger=logger)
if rc == 2:
# Partial History Fallback
logger.info("%s %s Fallback to brickfind" % (host, err.strip()))
# Exit only from process, handled in main.
sys.exit(rc)
elif rc != 0:
fail("%s - Change detection failed" % host, logger=logger)
if not localdir:
cmd_copy = [
"scp", "-i",
conf.get_opt("secret_pem"),
"root@%s:/%s" % (host, outfile),
os.path.dirname(outfile)
]
execute(cmd_copy,
exit_msg="%s - Copy command failed" % host,
logger=logger)
def mode_post(session_dir, args):
"""
If pre session file exists, overwrite session file
If pre session file does not exists, return ERROR
"""
status_file = os.path.join(session_dir, args.volume, "status")
logger.debug("Post is called - Session: %s, Volume: %s" %
(args.session, args.volume))
status_file_pre = status_file + ".pre"
if os.path.exists(status_file_pre):
os.rename(status_file_pre, status_file)
sys.exit(0)
else:
fail("Pre script is not run", logger=logger)
def mode_post(session_dir, args):
"""
If pre session file exists, overwrite session file
If pre session file does not exists, return ERROR
"""
status_file = os.path.join(session_dir, args.volume, "status")
logger.debug("Post is called - Session: %s, Volume: %s"
% (args.session, args.volume))
status_file_pre = status_file + ".pre"
if os.path.exists(status_file_pre):
os.rename(status_file_pre, status_file)
sys.exit(0)
else:
fail("Pre script is not run", logger=logger)
def _check_pools_size(self):
""""
Validate if pool and replicas have more than 8MiB (minimum pool size).
This function does not validate remote replicas sizes.
"""
size = 0
for part in self.parts:
size += part.size
if size < POOL_MIN_SIZE:
fail('The pool has to have at least 8 MiB')
for replica in self.replicas:
size = 0
for part in replica:
size += part.size
if size < POOL_MIN_SIZE:
fail('The pool has to have at least 8 MiB')
def _cs_request(self, url, method, **kwargs):
# Check that certain things are called correctly
if method in ["GET", "DELETE"]:
assert_not_in("body", kwargs)
elif method in ["PUT", "POST"]:
assert_in("body", kwargs)
# Call the method
munged_url = url.strip("/").replace("/", "_").replace(".", "_")
callback = "%s_%s" % (method.lower(), munged_url)
if not hasattr(self, callback):
fail("Called unknown API method: %s %s" % (method, url))
# Note the call
self.callstack.append((method, url, kwargs.get("body", None)))
status, body = getattr(self, callback)(**kwargs)
return httplib2.Response({"status": status}), body
