def post_servers_1234_action(self, body, **kw): assert_equal(len(body.keys()), 1) action = body.keys()[0] if action == 'reboot': assert_equal(body[action].keys(), ['type']) assert_in(body[action]['type'], ['HARD', 'SOFT']) elif action == 'rebuild': assert_equal(body[action].keys(), ['imageId']) elif action == 'resize': assert_equal(body[action].keys(), ['flavorId']) elif action == 'createBackup': assert_equal(set(body[action].keys()), set(['name', 'rotation', 'backup_type'])) elif action == 'confirmResize': assert_equal(body[action], None) # This one method returns a different response code return (204, None) elif action == 'revertResize': assert_equal(body[action], None) elif action == 'migrate': assert_equal(body[action], None) elif action == 'addFixedIp': assert_equal(body[action].keys(), ['networkId']) elif action == 'removeFixedIp': assert_equal(body[action].keys(), ['address']) else: fail("Unexpected server action: %s" % action) return (202, None)
def main(): # Start service try: l = launchd.Launchd() l.load(PLIST) except OSError, e: fail('Cannot load launchd job: {0}', e)
def gfid_to_path_using_batchfind(brick, gfids_file, output_file): """ find -samefile gets the inode number and crawls entire namespace to get the list of files/dirs having same inode number. Do find without any option, except the ignore directory option, print the output in <INODE_NUM> <PATH> format, use this output to look into in-memory dictionary of inode numbers got from the list of GFIDs """ with open(output_file, "a+") as fout: inode_dict = {} with open(gfids_file) as f: for gfid in f: gfid = gfid.strip() backend_path = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid) try: inode_dict[str(os.stat(backend_path).st_ino)] = 1 except (IOError, OSError) as e: if e.errno == ENOENT: continue else: fail("%s Failed to convert to path from " "GFID %s: %s" % (brick, gfid, e), logger=logger) if not inode_dict: return def inode_filter(path): try: st = os.lstat(path) except (OSError, IOError) as e: if e.errno == ENOENT: st = None else: raise if st and inode_dict.get(str(st.st_ino), None): return True return False brick_path_len = len(brick) def output_callback(path): path = path.strip() path = path[brick_path_len+1:] output_write(fout, path, args.output_prefix) ignore_dirs = [os.path.join(brick, dirname) for dirname in conf.get_opt("brick_ignore_dirs").split(",")] # Length of brick path, to remove from output path find(brick, callback_func=output_callback, filter_func=inode_filter, ignore_dirs=ignore_dirs) fout.flush() os.fsync(fout.fileno())
def post_servers_1234_action(self, body, **kw): assert_equal(len(body.keys()), 1) action = body.keys()[0] if action == "reboot": assert_equal(body[action].keys(), ["type"]) assert_in(body[action]["type"], ["HARD", "SOFT"]) elif action == "rebuild": assert_equal(body[action].keys(), ["imageId"]) elif action == "resize": assert_equal(body[action].keys(), ["flavorId"]) elif action == "confirmResize": assert_equal(body[action], None) # This one method returns a different response code return (204, None) elif action == "revertResize": assert_equal(body[action], None) elif action == "migrate": assert_equal(body[action], None) elif action == "addFixedIp": assert_equal(body[action].keys(), ["networkId"]) elif action == "removeFixedIp": assert_equal(body[action].keys(), ["address"]) else: fail("Unexpected server action: %s" % action) return (202, None)
def mode_pre(session_dir, args): global gtmpfilename """ Read from Session file and write to session.pre file """ endtime_to_update = int(time.time()) - get_changelog_rollover_time( args.volume) status_file = os.path.join(session_dir, args.volume, "status") status_file_pre = status_file + ".pre" mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger) # If Pre status file exists and running pre command again if os.path.exists(status_file_pre) and not args.regenerate_outfile: fail("Post command is not run after last pre, " "use --regenerate-outfile") start = 0 try: with open(status_file) as f: start = int(f.read().strip()) except ValueError: pass except (OSError, IOError) as e: fail("Error Opening Session file %s: %s" % (status_file, e), logger=logger) logger.debug("Pre is called - Session: %s, Volume: %s, " "Start time: %s, End time: %s" % (args.session, args.volume, start, endtime_to_update)) prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-") gtmpfilename = prefix + next(tempfile._get_candidate_names()) run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename) # Merger if args.full: cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile] execute(cmd, exit_msg="Failed to merge output files " "collected from nodes", logger=logger) else: # Read each Changelogs db and generate finaldb create_file(args.outfile, exit_on_err=True, logger=logger) outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles) write_output(args.outfile, outfilemerger, args.field_separator) try: os.remove(args.outfile + ".db") except (IOError, OSError): pass run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename) with open(status_file_pre, "w", buffering=0) as f: f.write(str(endtime_to_update)) sys.stdout.write("Generated output file %s\n" % args.outfile)
def validate_genesis(snapshot, genesis): step('Validating genesis') ddiff = DictDiffer(snapshot['genesis_state'], genesis) added = ddiff.added() removed = ddiff.removed() changed = ddiff.changed() if len(added) != 0 or len(removed) != 0 or len(changed) != 0: fail() if len(added) != 0: print "> Snapshot has '%s' and your genesis dont" % ( ','.join(added)) if len(removed) != 0: print "> Your genesis has %s and snapshot dont" % ( ','.join(removed)) if len(changed) != 0: print "> Your genesis and snapshot have different '%s'" % ( ','.join(changed)) print "# Snapshot genesis" print json.dumps(snapshot['genesis_state'], indent=2, sort_keys=True) print print "# Your genesis" print json.dumps(genesis, indent=2, sort_keys=True) return False success() return True
def askTime(self, sumSecTime): while True: strTime = input( "What's the time you want to show that picture at? (from 00:00 to 23:59, e.g. 22:30)\n" ) try: hours = int(strTime[0:2]) minutes = int(strTime[3:5]) seconds = (hours * 3600) + (minutes * 60) if (0 <= hours <= 23 and 0 <= minutes <= 59 and strTime[2:3] == ":"): if (self.isLater(sumSecTime, seconds)): break else: utils.fail("NameError") else: #crash try() and force user to try again utils.fail("ValueError") except ValueError: print("Invalid time format! Try again") time.sleep(2) except NameError: print( "This image must be set later than the last one! Try again" ) time.sleep(2) print() return strTime
def main(): args = _get_args() mkdirp(conf.get_opt("session_dir"), exit_on_err=True) if args.mode == "list": session_dir = conf.get_opt("session_dir") else: session_dir = os.path.join(conf.get_opt("session_dir"), args.session) if not os.path.exists(session_dir) and args.mode not in ["create", "list"]: fail("Invalid session %s" % args.session) vol_dir = os.path.join(session_dir, args.volume) if not os.path.exists(vol_dir) and args.mode not in ["create", "list"]: fail("Session %s not created with volume %s" % (args.session, args.volume)) mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume), exit_on_err=True) log_file = os.path.join(conf.get_opt("log_dir"), args.session, args.volume, "cli.log") setup_logger(logger, log_file, args.debug) # globals() will have all the functions already defined. # mode_<args.mode> will be the function name to be called globals()["mode_" + args.mode](session_dir, args)
def info(): """获取用户信息 --- tags: - 用户 responses: 200: description: 获取成功 schema: type: object properties: code: type: int data: type: array $ref: '#/definitions/Module' message: type: string examples: code: 0 data: [{}, {}] message: 'success' """ token = request.args.get('token') data = verify_token(token) if not data: return fail(401) user = Admin.query.get_or_404(data['user_id']) if not user: return fail(401) res = {'data': {'name': user.name, 'avatar': user.avatar}} return success(res)
def export(self, targz: str, **kwargs: dict): if not os.path.isdir(self.path): utils.fail("%s: VM does not exist" % self.name) exit(1) if not targz: targz = '%s.tar.gz' % self.name targz = os.path.abspath(targz) utils.info("exporting to %s" % targz) tar = '%s/%s' % (self.path, os.path.basename(os.path.splitext(targz)[0])) if os.path.exists(targz): utils.fail("%s: package already exists" % targz) exit(1) os.chdir(self.path) # create intermediate files in VM's path utils.pend("package disk image") utils.execute(['tar', '-cSf', tar, os.path.basename(self.path_raw)]) utils.ok() for s in self.snapshots: utils.pend("package snapshot: %s" % s) local_snapshot = '%s.%s' % (os.path.basename(self.path_raw), s) utils.execute(['tar', '-rf', tar, local_snapshot]) utils.ok() utils.pend("compress package", msg="may take some time") utils.execute(['gzip', '-c', tar], outfile=targz) utils.ok() utils.pend("clean up") os.unlink(tar) utils.ok() self.scan_snapshots()
def exec(self, cmd, args='', expected_exit=0): """Execute binary in current test context""" env = {**self.env, **os.environ.copy(), **self.test.utenv} if sys.platform == 'win32': env['PATH'] = self.build.libdir + os.pathsep +\ envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\ env.get('PATH', '') cmd = os.path.join(self.build.exedir, cmd) + '.exe' else: if self.test.ld_preload: env['LD_PRELOAD'] = env.get('LD_PRELOAD', '') + os.pathsep +\ self.test.ld_preload self.valgrind.handle_ld_preload(self.test.ld_preload) env['LD_LIBRARY_PATH'] = self.build.libdir + os.pathsep +\ envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\ env.get('LD_LIBRARY_PATH', '') cmd = os.path.join(self.test.cwd, cmd) + self.build.exesuffix cmd = '{} {}'.format(self.valgrind.cmd, cmd) proc = sp.run('{} {}'.format(cmd, args), env=env, cwd=self.test.cwd, shell=True, timeout=self.conf.timeout, stdout=sp.PIPE, stderr=sp.STDOUT, universal_newlines=True) if sys.platform != 'win32' and expected_exit == 0 \ and not self.valgrind.validate_log(): self.test.fail(proc.stdout) if proc.returncode != expected_exit: fail(proc.stdout, exit_code=proc.returncode) else: self.msg.print_verbose(proc.stdout)
def get_nodes(volume): """ Get the gluster volume info xml output and parse to get the brick details. """ global vol_statusStr cmd = ["gluster", 'volume', 'info', volume, "--xml"] _, data, _ = execute(cmd, exit_msg="Failed to Run Gluster Volume Info", logger=logger) tree = etree.fromstring(data) # Test to check if volume has been deleted after session creation count_el = tree.find('volInfo/volumes/count') if int(count_el.text) == 0: fail("Unable to get volume details", logger=logger) # this status is used in caller: run_cmd_nodes vol_statusStr = tree.find('volInfo/volumes/volume/statusStr').text nodes = [] volume_el = tree.find('volInfo/volumes/volume') try: for b in volume_el.findall('bricks/brick'): nodes.append((b.find('hostUuid').text, b.find('name').text)) except (ParseError, AttributeError, ValueError) as e: fail("Failed to parse Volume Info: %s" % e, logger=logger) return nodes
def mode_create(session_dir, args): validate_session_name(args.session) logger.debug("Init is called - Session: %s, Volume: %s" % (args.session, args.volume)) mkdirp(session_dir, exit_on_err=True, logger=logger) mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True, logger=logger) status_file = os.path.join(session_dir, args.volume, "status") if os.path.exists(status_file) and not args.force: fail("Session %s already created" % args.session, logger=logger) if not os.path.exists(status_file) or args.force: ssh_setup(args) enable_volume_options(args) # Add Rollover time to current time to make sure changelogs # will be available if we use this time as start time time_to_update = int(time.time()) + get_changelog_rollover_time( args.volume) run_cmd_nodes("create", args, time_to_update=str(time_to_update)) if not os.path.exists(status_file) or args.reset_session_time: with open(status_file, "w") as f: f.write(str(time_to_update)) sys.stdout.write("Session %s created with volume %s\n" % (args.session, args.volume)) sys.exit(0)
def _run_match(self): """Match log files""" cwd_listdir = [path.join(self.cwd, f) for f in listdir(self.cwd)] suffix = '{}.log.match'.format(self.testnum) def is_matchfile(f): """Match file ends with specific suffix and a char before suffix is not a digit""" before_suffix = -len(suffix) - 1 return path.isfile(f) and f.endswith(suffix) and \ not f[before_suffix].isdigit() match_files = filter(is_matchfile, cwd_listdir) prefix = 'perl ' if sys.platform == 'win32' else '' match_cmd = prefix + path.join(hlp.ROOTDIR, 'match') for mf in match_files: cmd = '{} {}'.format(match_cmd, mf) proc = sp.run(cmd.split(), stdout=sp.PIPE, cwd=self.cwd, stderr=sp.STDOUT, universal_newlines=True) if proc.returncode != 0: fail(proc.stdout, exit_code=proc.returncode) else: self.msg.print_verbose(proc.stdout)
def main(test_filter=None): if not test_filter: test_filter = "*" try: test_suite = list(TestParser.read_all(test_filter)) except (UnknownFieldError, RequiredFieldError) as error: print(fail(str(error))) exit(1) total_tests = 0 for fname, suite_tests in test_suite: total_tests += len(suite_tests) failed_tests = [] print( ok("[==========]") + " Running %d tests from %d test cases.\n" % (total_tests, len(test_suite))) start_time = time.time() skipped_tests = [] for fname, tests in test_suite: print(ok("[----------]") + " %d tests from %s" % (len(tests), fname)) for test in tests: status = Utils.run_test(test) if Utils.skipped(status): skipped_tests.append((fname, test, status)) if Utils.failed(status): failed_tests.append("%s.%s" % (fname, test.name)) # TODO(mmarchini) elapsed time per test suite and per test (like gtest) print(ok("[----------]") + " %d tests from %s\n" % (len(tests), fname)) elapsed = time.time() - start_time total_tests -= len(skipped_tests) # TODO(mmarchini) pretty print time print( ok("[==========]") + " %d tests from %d test cases ran. (%s total)" % (total_tests, len(test_suite), elapsed)) print( ok("[ PASSED ]") + " %d tests." % (total_tests - len(failed_tests))) if skipped_tests: print( warn("[ SKIP ]") + " %d tests, listed below:" % len(skipped_tests)) for test_suite, test, status in skipped_tests: print( warn("[ SKIP ]") + " %s.%s (%s)" % (test_suite, test.name, Utils.skip_reason(test, status))) if failed_tests: print( fail("[ FAILED ]") + " %d tests, listed below:" % len(failed_tests)) for failed_test in failed_tests: print(fail("[ FAILED ]") + " %s" % failed_test) if failed_tests: exit(1)
async def upimg(request): # 判断用户是否具有上传权限 if request.headers.get('token') != TOKEN: return fail('您没有使用本服务的权限') # 判断参数是否正确 if not request.files and not request.files.get('file'): return fail('error args') image = request.files.get('file').body # 判断文件是否支持 imageSuffix = getSuffix(bytes2hex(image)) if 'error' in imageSuffix: return fail(imageSuffix) # 组织图片存储路径 m1 = hashlib.md5() m1.update(image) md5Name = m1.hexdigest() saveDir = baseDir + md5Name[0:2] + '/' savePath = saveDir + md5Name[2:] + '.' + imageSuffix resPath = '/' + md5Name[0:2] + '/' + md5Name[2:] + '.' + imageSuffix # 如果文件夹不存在,就创建文件夹 if not os.path.exists(saveDir): os.makedirs(saveDir) # 将文件写入到硬盘 tempFile = open(savePath, 'wb') tempFile.write(image) tempFile.close() # 给客户端返回结果 return ok({"path": resPath})
def _run_match(self): """Match log files""" cwd_listdir = [path.join(self.cwd, f) for f in os.listdir(self.cwd)] suffix = '{}.log.match'.format(self.testnum) def is_matchfile(f): """Match file ends with specific suffix and a char before suffix is not a digit""" before_suffix = -len(suffix) - 1 return path.isfile(f) and f.endswith(suffix) and \ not f[before_suffix].isdigit() match_files = filter(is_matchfile, cwd_listdir) prefix = 'perl ' if sys.platform == 'win32' else '' match_cmd = prefix + path.join(hlp.ROOTDIR, 'match') for mf in match_files: cmd = '{} {}'.format(match_cmd, mf) proc = sp.run(cmd.split(), stdout=sp.PIPE, cwd=self.cwd, stderr=sp.STDOUT, universal_newlines=True) if proc.returncode != 0: fail(proc.stdout, exit_code=proc.returncode) else: self.msg.print_verbose(proc.stdout)
def mode_pre(session_dir, args): """ Read from Session file and write to session.pre file """ endtime_to_update = int(time.time()) - int( conf.get_opt("changelog_rollover_time")) status_file = os.path.join(session_dir, args.volume, "status") status_file_pre = status_file + ".pre" mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger) start = 0 try: with open(status_file) as f: start = int(f.read().strip()) except ValueError: pass except (OSError, IOError) as e: fail("Error Opening Session file %s: %s" % (status_file, e), logger=logger) logger.debug("Pre is called - Session: %s, Volume: %s, " "Start time: %s, End time: %s" % (args.session, args.volume, start, endtime_to_update)) run_in_nodes(args.volume, start, args) with open(status_file_pre, "w", buffering=0) as f: f.write(str(endtime_to_update)) sys.stdout.write("Generated output file %s\n" % args.outfile)
def get_nodes(volume): """ Get the gluster volume info xml output and parse to get the brick details. """ global vol_statusStr; cmd = ["gluster", 'volume', 'info', volume, "--xml"] _, data, _ = execute(cmd, exit_msg="Failed to Run Gluster Volume Info", logger=logger) tree = etree.fromstring(data) # Test to check if volume has been deleted after session creation count_el = tree.find('volInfo/volumes/count') if int(count_el.text) == 0: fail("Unable to get volume details", logger=logger) # this status is used in caller: run_cmd_nodes vol_statusStr = tree.find('volInfo/volumes/volume/statusStr').text nodes = [] volume_el = tree.find('volInfo/volumes/volume') try: for b in volume_el.findall('bricks/brick'): nodes.append((b.find('hostUuid').text, b.find('name').text)) except (ParseError, AttributeError, ValueError) as e: fail("Failed to parse Volume Info: %s" % e, logger=logger) return nodes
def parse_acquisitions(self): src_paths = [] for f in self._files: # Check if it is a DICOM try: ds = dicom.read_file(f) except InvalidDicomError: utils.warning('Not a DICOM file: {}'.format(f)) continue # Filter by serie description dsDesc = ds.SeriesDescription if any(_ in dsDesc for _ in self.exclusionCriteria): self._excluded.add(dsDesc) else: src_paths.append(f) groups = dcmstack.parse_and_group(src_paths) for key, group in groups.iteritems(): if self._isFromOneDirectory(group): #utils.ok("Same directory: {}".format(key[2])) stack = dcmstack.stack_group(group) inDir = os.path.dirname(group[0][2]) self._acquisitions.append( Acquisition(inDir, stack, self._session)) else: #TODO: regroup dicoms in a tmp directory utils.new_line() utils.fail( "DICOM of '{}' are not in the same directory.\nThis structure won't be compatible with dcm2niibatch" .format(key[2]))
def read(self, arg): newresult = {} f = open(self.arg_id(arg), "r") for line in f.readlines(): line = line.strip().split() if len(line) % 2 == 1 or len(line) < 2: # Fail utils.fail(self.arg_id(arg)) f.close() return new_key = {k: v for k, v in arg} # parse line for i in xrange(0, len(line) - 2, 2): i_name = line[i] i_value = float(line[i + 1]) new_key[i_name] = i_value new_key["name"] = line[-2] newresult[utils.dict2tuple(new_key)] = float(line[-1]) # Merge result with newresult for k in newresult.keys(): self.result[k] = newresult[k] f.close()
def main(): # Bind to service port server_port = mach.BootstrapServer.lookup("org.freenas.test.mach.ipc-server") local_port = mach.Port() print 'Service port: {0}'.format(server_port) print 'Local port: {0}'.format(local_port) # Send a few messages for i in range(0, 100): msg = mach.Message() msg.bits = mach.make_msg_bits( mach.MessageType.MACH_MSG_TYPE_COPY_SEND, mach.MessageType.MACH_MSG_TYPE_MAKE_SEND ) msg.body = bytearray(random_str()) local_port.send(server_port, msg) reply = local_port.receive() print 'Received reply: {0}'.format(reply.body) if reply.body != msg.body: fail('Reply mismatch: {0} != {1}'.format(msg.body, reply.body)) # Exit msg = mach.Message() msg.bits = mach.make_msg_bits( mach.MessageType.MACH_MSG_TYPE_COPY_SEND, mach.MessageType.MACH_MSG_TYPE_MAKE_SEND ) msg.body = bytearray('EXIT') mach.null_port.send(server_port, msg)
def main(): # Create send port try: send = mach.Port() send.insert_right(mach.MessageType.MACH_MSG_TYPE_MAKE_SEND) print 'Send port: {0}'.format(send) except mach.MachException, e: fail('Cannot create send port: {0}'.format(e))
def vm_init(path: str): utils.pend("initialise VM directory: %s" % path) try: os.mkdir(path) except OSError as e: utils.fail(e.strip()) exit(1) utils.ok()
def mode_list(session_dir, args): """ List available sessions to stdout, if session name is set only list that session. """ if args.session: if not os.path.exists(os.path.join(session_dir, args.session)): fail("Invalid Session", logger=logger) sessions = [args.session] else: sessions = [] for d in os.listdir(session_dir): if d != ".keys": sessions.append(d) output = [] for session in sessions: # Session Volume Last Processed volnames = os.listdir(os.path.join(session_dir, session)) for volname in volnames: if args.volume and args.volume != volname: continue status_file = os.path.join(session_dir, session, volname, "status") last_processed = None try: with open(status_file) as f: last_processed = f.read().strip() except (OSError, IOError) as e: if e.errno == ENOENT: continue else: raise output.append((session, volname, last_processed)) if output: sys.stdout.write("%s %s %s\n" % ("SESSION".ljust(25), "VOLUME".ljust(25), "SESSION TIME".ljust(25))) sys.stdout.write("-" * 75) sys.stdout.write("\n") for session, volname, last_processed in output: sess_time = 'Session Corrupted' if last_processed: try: sess_time = human_time(last_processed) except TypeError: sess_time = 'Session Corrupted' sys.stdout.write( "%s %s %s\n" % (session.ljust(25), volname.ljust(25), sess_time.ljust(25))) if not output: if args.session or args.volume: fail("Invalid Session", logger=logger) else: sys.stdout.write("No sessions found.\n")
def mode_list(session_dir, args): """ List available sessions to stdout, if session name is set only list that session. """ if args.session: if not os.path.exists(os.path.join(session_dir, args.session)): fail("Invalid Session", logger=logger) sessions = [args.session] else: sessions = [] for d in os.listdir(session_dir): if d != ".keys": sessions.append(d) output = [] for session in sessions: # Session Volume Last Processed volnames = os.listdir(os.path.join(session_dir, session)) for volname in volnames: if args.volume and args.volume != volname: continue status_file = os.path.join(session_dir, session, volname, "status") last_processed = None try: with open(status_file) as f: last_processed = f.read().strip() except (OSError, IOError) as e: if e.errno == ENOENT: pass else: raise output.append((session, volname, last_processed)) if output: sys.stdout.write("%s %s %s\n" % ("SESSION".ljust(25), "VOLUME".ljust(25), "SESSION TIME".ljust(25))) sys.stdout.write("-"*75) sys.stdout.write("\n") for session, volname, last_processed in output: sess_time = 'Session Corrupted' if last_processed: try: sess_time = human_time(last_processed) except TypeError: sess_time = 'Session Corrupted' sys.stdout.write("%s %s %s\n" % (session.ljust(25), volname.ljust(25), sess_time.ljust(25))) if not output: if args.session or args.volume: fail("Invalid Session", logger=logger) else: sys.stdout.write("No sessions found.\n")
def mode_create(session_dir, args): logger.debug("Init is called - Session: %s, Volume: %s" % (args.session, args.volume)) execute(["gluster", "volume", "info", args.volume], exit_msg="Unable to get volume details", logger=logger) mkdirp(session_dir, exit_on_err=True, logger=logger) mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True, logger=logger) status_file = os.path.join(session_dir, args.volume, "status") if os.path.exists(status_file) and not args.force: fail("Session %s already created" % args.session, logger=logger) if not os.path.exists(status_file) or args.force: ssh_setup(args) execute(["gluster", "volume", "set", args.volume, "build-pgfid", "on"], exit_msg="Failed to set volume option build-pgfid on", logger=logger) logger.info("Volume option set %s, build-pgfid on" % args.volume) execute([ "gluster", "volume", "set", args.volume, "changelog.changelog", "on" ], exit_msg="Failed to set volume option " "changelog.changelog on", logger=logger) logger.info("Volume option set %s, changelog.changelog on" % args.volume) execute([ "gluster", "volume", "set", args.volume, "changelog.capture-del-path", "on" ], exit_msg="Failed to set volume option " "changelog.capture-del-path on", logger=logger) logger.info("Volume option set %s, changelog.capture-del-path on" % args.volume) # Add Rollover time to current time to make sure changelogs # will be available if we use this time as start time time_to_update = int(time.time()) + get_changelog_rollover_time( args.volume) run_cmd_nodes("create", args, time_to_update=str(time_to_update)) if not os.path.exists(status_file) or args.reset_session_time: with open(status_file, "w", buffering=0) as f: f.write(str(time_to_update)) sys.exit(0)
def node_cmd(host, host_uuid, task, cmd, args, opts): """ Runs command via ssh if host is not local """ try: localdir = is_host_local(host_uuid) # this is so to avoid deleting the ssh keys on local node which # otherwise cause ssh password prompts on the console (race conditions) # mode_delete() should be cleaning up the session tree if localdir and task == "delete": return pem_key_path = get_pem_key_path(args.session, args.volume) if not localdir: # prefix with ssh command if not local node cmd = [ "ssh", "-oNumberOfPasswordPrompts=0", "-oStrictHostKeyChecking=no", # We force TTY allocation (-t -t) so that Ctrl+C is handed # through; see: # https://bugzilla.redhat.com/show_bug.cgi?id=1382236 # Note that this turns stderr of the remote `cmd` # into stdout locally. "-t", "-t", "-i", pem_key_path, "root@%s" % host ] + cmd (returncode, err, out) = execute(cmd, logger=logger) if returncode != 0: # Because the `-t -t` above turns the remote stderr into # local stdout, we need to log both stderr and stdout # here to print all error messages. fail("%s - %s failed; stdout (including remote stderr):\n" "%s\n" "stderr:\n" "%s" % (host, task, out, err), returncode, logger=logger) if opts.get("copy_outfile", False) and not localdir: cmd_copy = [ "scp", "-oNumberOfPasswordPrompts=0", "-oStrictHostKeyChecking=no", "-i", pem_key_path, "root@%s:/%s" % (host, opts.get("node_outfile")), os.path.dirname(opts.get("node_outfile")) ] execute(cmd_copy, exit_msg="%s - Copy command failed" % host, logger=logger) except KeyboardInterrupt: sys.exit(2)
def main() -> None: for func in utils.get_functions(): if not func.decomp_name: continue if func.status == utils.FunctionStatus.Matching: if not check_function(func.addr, func.size, func.decomp_name): utils.fail(f"{func.decomp_name} was marked as matching but does not match") return
def open_output(self): try: output_file = self.output + ("/out-%04i.txt" % self.number) self.fd = open(output_file, "w") except Exception as e: print("") from utils import fail fail("Could not open task output file: " + output_file + "\n" + str(e))
def dump_fn(name: str) -> None: expected_dir = root / "expected" try: fn = get_fn_from_my_elf(name) path = expected_dir / f"{name}.bin" path.parent.mkdir(exist_ok=True) path.write_bytes(fn) except KeyError: utils.fail("could not find function")
def set_parts(self, *parts): """ Adds a main pool to the poolset file. The function accepts a list of Parts of any length. Should not be called more than once. """ if not self.parts: self.parts = list(parts) else: fail('This function should not be called more than once.')
def import_raw(self, raw: str, force: bool): if not os.path.exists(raw): utils.fail("%s: file not found" % raw) exit(1) self.initialise(force) utils.pend("copy disk image") utils.execute(['cp', raw, self.path_raw]) utils.ok()
def setValues(self, root: Node): if (len(root.childs) == 0): if (root.info in self.eval_config): root.truth = self.eval_config[root.info] else: fail("404", (root.info, )) else: for child in root.childs: self.setValues(child) self.was_set = True
def __init__(self, path_yaml: str): if not os.path.isfile(path_yaml): utils.fail("%s: file not found" % path_yaml) exit(1) yaml = Batch.YAML(path_yaml) self.variables = yaml.tree['variables'] self.commands = [] for ctoken, i in zip(yaml.tree['commands'], range(len(yaml.tree['commands']))): c = Batch.Command(ctoken, self.variables) self.commands.append(c)
def validate_volume(volume): cmd = ["gluster", 'volume', 'info', volume, "--xml"] _, data, _ = execute(cmd, exit_msg="Failed to Run Gluster Volume Info", logger=logger) try: tree = etree.fromstring(data) statusStr = tree.find('volInfo/volumes/volume/statusStr').text except (ParseError, AttributeError) as e: fail("Invalid Volume: Check the Volume name! %s" % e) if statusStr != "Started": fail("Volume %s is not online" % volume)
def update_evaluator(self, config: dict): for k in config: v = config[k] if (v == "True"): config[k] = True elif (v == "False"): config[k] = False else: from utils import fail fail("config", (k, v)) self.evaluator.eval_config = config
def node_cmd(host, host_uuid, task, cmd, args, opts): """ Runs command via ssh if host is not local """ try: localdir = is_host_local(host_uuid) # this is so to avoid deleting the ssh keys on local node which # otherwise cause ssh password prompts on the console (race conditions) # mode_delete() should be cleaning up the session tree if localdir and task == "delete": return pem_key_path = get_pem_key_path(args.session, args.volume) if not localdir: # prefix with ssh command if not local node cmd = ["ssh", "-oNumberOfPasswordPrompts=0", "-oStrictHostKeyChecking=no", # We force TTY allocation (-t -t) so that Ctrl+C is handed # through; see: # https://bugzilla.redhat.com/show_bug.cgi?id=1382236 # Note that this turns stderr of the remote `cmd` # into stdout locally. "-t", "-t", "-i", pem_key_path, "root@%s" % host] + cmd (returncode, err, out) = execute(cmd, logger=logger) if returncode != 0: # Because the `-t -t` above turns the remote stderr into # local stdout, we need to log both stderr and stdout # here to print all error messages. fail("%s - %s failed; stdout (including remote stderr):\n" "%s\n" "stderr:\n" "%s" % (host, task, out, err), returncode, logger=logger) if opts.get("copy_outfile", False) and not localdir: cmd_copy = ["scp", "-oNumberOfPasswordPrompts=0", "-oStrictHostKeyChecking=no", "-i", pem_key_path, "root@%s:/%s" % (host, opts.get("node_outfile")), os.path.dirname(opts.get("node_outfile"))] execute(cmd_copy, exit_msg="%s - Copy command failed" % host, logger=logger) except KeyboardInterrupt: sys.exit(2)
def check_depth(self, root: Node): """Check the depth, will call the recursive one Arguments: root {Node} -- The starting node """ if (self.expected_depth[0] == 0): self.get_expected_depth(self.input_phrase) ch = self.check_depth_rec(root, 1) if (ch != self.expected_depth[0] or 0 != self.expected_depth[1] or root.childs[0].info == ""): fail("ret", (-1, ))
def execute(args, cmd_line): # Informative: ip = utils.get_default_ip() if args['headless']: utils.info("VNC: port %d (connect with `$vncclient %s:%d`)" % (args['vnc_port'], ip, args['vnc_display'])) utils.info("Qemu monitor: port %d (connect with `{nc,telnet} %s %d)" % (args['monitor_port'], ip, args['monitor_port'])) if args['mode'] == 'sym': utils.info("Experiment name: %s" % args['expname']) if args['script'] or args['command']: utils.info("Watchdog: port %d" % args['command_port']) utils.debug("Command line:\n%s" % ' '.join(cmd_line)) if args['dry_run']: exit(1) environ = dict(os.environ) if args['mode'] == 'sym': environ['LUA_PATH'] = ';'.join(['%s/?.lua' % args['config_root'], environ.get('LUA_PATH', '')]) utils.debug("LUA_PATH=%s" % environ['LUA_PATH']) # each experiment gets its own directory: try: utils.pend("Creating experiment directory %s" % args['exppath']) os.makedirs(args['exppath']) utils.ok() except FileExistsError: utils.fail("Experiment %s already exists. Please choose another name." % args['expname']) exit(1) if args['timeout']: kill_me_later(args['timeout']) obj = None if args['script']: module_file, test = args['script'] with open(module_file, 'r') as f: code = f.read() obj = Script(code=code, test=test) if args['command']: obj = Command.from_cmd_args(args['command'], args['env_var'] or []) if obj: async_send_command(obj, 'localhost', args['command_port'], TIMEOUT_CMD) # drop `s2e-last` symlink somewhere where it does not get in the way: os.chdir(utils.CHEFROOT_EXPDATA) os.execvpe(cmd_line[0], cmd_line, environ)
def mode_create(session_dir, args): logger.debug("Init is called - Session: %s, Volume: %s" % (args.session, args.volume)) execute(["gluster", "volume", "info", args.volume], exit_msg="Unable to get volume details", logger=logger) mkdirp(session_dir, exit_on_err=True, logger=logger) mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True, logger=logger) status_file = os.path.join(session_dir, args.volume, "status") if os.path.exists(status_file) and not args.force: fail("Session %s already created" % args.session, logger=logger) if not os.path.exists(status_file) or args.force: ssh_setup(args) execute(["gluster", "volume", "set", args.volume, "build-pgfid", "on"], exit_msg="Failed to set volume option build-pgfid on", logger=logger) logger.info("Volume option set %s, build-pgfid on" % args.volume) execute(["gluster", "volume", "set", args.volume, "changelog.changelog", "on"], exit_msg="Failed to set volume option " "changelog.changelog on", logger=logger) logger.info("Volume option set %s, changelog.changelog on" % args.volume) execute(["gluster", "volume", "set", args.volume, "changelog.capture-del-path", "on"], exit_msg="Failed to set volume option " "changelog.capture-del-path on", logger=logger) logger.info("Volume option set %s, changelog.capture-del-path on" % args.volume) # Add Rollover time to current time to make sure changelogs # will be available if we use this time as start time time_to_update = int(time.time()) + get_changelog_rollover_time( args.volume) run_cmd_nodes("create", args, time_to_update=str(time_to_update)) if not os.path.exists(status_file) or args.reset_session_time: with open(status_file, "w", buffering=0) as f: f.write(str(time_to_update)) sys.exit(0)
def ssh_setup(args): pem_key_path = get_pem_key_path(args.session, args.volume) if not os.path.exists(pem_key_path): # Generate ssh-key cmd = ["ssh-keygen", "-N", "", "-f", pem_key_path] execute(cmd, exit_msg="Unable to generate ssh key %s" % pem_key_path, logger=logger) logger.info("Ssh key generated %s" % pem_key_path) try: shutil.copyfile(pem_key_path + ".pub", os.path.join(conf.get_opt("session_dir"), ".keys", "%s_%s_secret.pem.pub" % (args.session, args.volume))) except (IOError, OSError) as e: fail("Failed to copy public key to %s: %s" % (os.path.join(conf.get_opt("session_dir"), ".keys"), e), logger=logger) # Copy pub file to all nodes cmd = ["gluster", "system::", "copy", "file", "/glusterfind/.keys/%s.pub" % os.path.basename(pem_key_path)] execute(cmd, exit_msg="Failed to distribute ssh keys", logger=logger) logger.info("Distributed ssh key to all nodes of Volume") # Add to authorized_keys file in each node cmd = ["gluster", "system::", "execute", "add_secret_pub", "root", "/glusterfind/.keys/%s.pub" % os.path.basename(pem_key_path)] execute(cmd, exit_msg="Failed to add ssh keys to authorized_keys file", logger=logger) logger.info("Ssh key added to authorized_keys of Volume nodes")
def check(self, root: Node): if (root.info == ""): return False if len(root.childs) > 2: fail("syntax", (root, )) return False for child in root.childs: if (self.check_childs(child)): self.check(child) else: fail("syntax", (child, )) return False return True
def node_run(volume, host, path, start, outfile, args, fallback=False): """ If host is local node, execute the command locally. If not local execute the CHANGE_DETECTOR command via ssh and copy the output file from remote node using scp. """ localdir = is_host_local(host) pem_key_path = get_pem_key_path(args.session, args.volume) # If Full backup is requested or start time is zero, use brickfind change_detector = conf.get_change_detector(args.change_detector) if ((start == 0 or args.full) and args.change_detector == "changelog") or \ fallback: change_detector = conf.get_change_detector("brickfind") # CHANGE_DETECTOR <SESSION> <VOLUME> <BRICK> <OUTFILE> <START> --debug # --gfidpath <TYPE> cmd = [change_detector, args.session, volume, path, outfile, str(start), "--output-prefix", args.output_prefix] + \ (["--debug"] if args.debug else []) + \ (["--full"] if args.full else []) if not localdir: # prefix with ssh command if not local node cmd = ["ssh", "-i", pem_key_path, "root@%s" % host] + cmd rc, out, err = execute(cmd, logger=logger) if rc == 2: # Partial History Fallback logger.info("%s %s Fallback to brickfind" % (host, err.strip())) # Exit only from process, handled in main. sys.exit(rc) elif rc != 0: fail("%s - Change detection failed" % host, logger=logger) if not localdir: cmd_copy = ["scp", "-i", pem_key_path, "root@%s:/%s" % (host, outfile), os.path.dirname(outfile)] execute(cmd_copy, exit_msg="%s - Copy command failed" % host, logger=logger)
def delete(self, snapshot: str=None, **kwargs: dict): if snapshot: utils.pend("delete snapshot %s" % kwargs['name[:snapshot]']) if snapshot not in self.snapshots: utils.fail("%s: snapshot does not exist" % kwargs['name[:snapshot]']) exit(1) try: os.unlink('%s.%s' % (self.path_raw, snapshot)) utils.ok() except PermissionError: utils.fail("Permission denied") exit(1) else: if not os.path.isdir(self.path): utils.fail("%s: VM does not exist" % self.name) exit(1) if not utils.ask("Delete VM %s?" % self.name, default=False): exit(1) utils.pend("delete %s" % self.name) try: shutil.rmtree(self.path) utils.ok() except PermissionError: utils.fail("Permission denied") exit(1)
def node_run(volume, host, path, start, outfile, args, fallback=False): """ If host is local node, execute the command locally. If not local execute the CHANGE_DETECTOR command via ssh and copy the output file from remote node using scp. """ localdir = is_host_local(host) # If Full backup is requested or start time is zero, use brickfind change_detector = conf.get_change_detector(args.change_detector) if ((start == 0 or args.full) and args.change_detector == "changelog") or \ fallback: change_detector = conf.get_change_detector("brickfind") # CHANGE_DETECTOR <SESSION> <VOLUME> <BRICK> <OUTFILE> <START> --debug # --gfidpath <TYPE> cmd = [change_detector, args.session, volume, path, outfile, str(start), "--output-prefix", args.output_prefix] + \ (["--debug"] if args.debug else []) + \ (["--full"] if args.full else []) if not localdir: # prefix with ssh command if not local node cmd = ["ssh", "-i", conf.get_opt("secret_pem"), "root@%s" % host] + cmd rc, out, err = execute(cmd, logger=logger) if rc == 2: # Partial History Fallback logger.info("%s %s Fallback to brickfind" % (host, err.strip())) # Exit only from process, handled in main. sys.exit(rc) elif rc != 0: fail("%s - Change detection failed" % host, logger=logger) if not localdir: cmd_copy = [ "scp", "-i", conf.get_opt("secret_pem"), "root@%s:/%s" % (host, outfile), os.path.dirname(outfile) ] execute(cmd_copy, exit_msg="%s - Copy command failed" % host, logger=logger)
def mode_post(session_dir, args): """ If pre session file exists, overwrite session file If pre session file does not exists, return ERROR """ status_file = os.path.join(session_dir, args.volume, "status") logger.debug("Post is called - Session: %s, Volume: %s" % (args.session, args.volume)) status_file_pre = status_file + ".pre" if os.path.exists(status_file_pre): os.rename(status_file_pre, status_file) sys.exit(0) else: fail("Pre script is not run", logger=logger)
def _check_pools_size(self): """" Validate if pool and replicas have more than 8MiB (minimum pool size). This function does not validate remote replicas sizes. """ size = 0 for part in self.parts: size += part.size if size < POOL_MIN_SIZE: fail('The pool has to have at least 8 MiB') for replica in self.replicas: size = 0 for part in replica: size += part.size if size < POOL_MIN_SIZE: fail('The pool has to have at least 8 MiB')
def _cs_request(self, url, method, **kwargs): # Check that certain things are called correctly if method in ["GET", "DELETE"]: assert_not_in("body", kwargs) elif method in ["PUT", "POST"]: assert_in("body", kwargs) # Call the method munged_url = url.strip("/").replace("/", "_").replace(".", "_") callback = "%s_%s" % (method.lower(), munged_url) if not hasattr(self, callback): fail("Called unknown API method: %s %s" % (method, url)) # Note the call self.callstack.append((method, url, kwargs.get("body", None))) status, body = getattr(self, callback)(**kwargs) return httplib2.Response({"status": status}), body