def wrapper(cmd, *args, **kwargs): out = f(cmd + ' 2>&1 /dev/null', *args, **kwargs) if out.return_code != 0: logger.critical( "Command '{}' returned error code {} with the following error:\n" .format(cmd, out.return_code) + '\n' + out) raise Exception("Command '{}' failed.".format(cmd)) logger.debug(out)
def wrapper(cmd, *args, **kwargs): out = f(cmd + ' 2>&1 /dev/null', *args, **kwargs) if out.return_code != 0: filtered = [] for line in out.split('\n'): if any([line.startswith(s) for s in ['cp ', 'mkdir ', ' CC', ' AR']]) or 'warning' in line: continue filtered.append(line) logger.critical("Command '{}' returned error code {} with the following error:\n" .format(cmd, out.return_code) + '\n'.join(filtered)) raise Exception("Command '{}' failed.".format(cmd))
def __remake(name, build=False, **kwargs): """ Remake the malicious mote of an experiment. (meaning that it lets all simulation's files unchanged except ./motes/malicious.[target]) :param name: experiment name :param path: expanded path of the experiment (dynamically filled in through 'command' decorator with 'expand' """ path = kwargs['path'] logger.debug(" > Retrieving parameters...") params = read_config(path) ext_lib = params.get("ext_lib") if ext_lib and not exists(ext_lib): logger.error("External library does not exist !") logger.critical("Make aborted.") return False logger.debug(" > Recompiling malicious mote...") # remove former compiled malicious mote and prepare the template templates = get_path(path, 'templates', create=True) get_path(templates, 'motes', create=True) copy_files((TEMPLATES_FOLDER, 'experiment'), templates, ('motes/{}.c'.format(params["mtype_malicious"]), 'motes/malicious.c')) # recreate malicious C file from template and clean the temporary template replacements = render_templates(path, only_malicious=True, **params) # then clean the temporary folder with templates remove_folder(templates) # now recompile with settings(hide(*HIDDEN_ALL), warn_only=True): with_malicious = join(path, 'with-malicious', 'motes') without_malicious = join(path, 'without-malicious', 'motes') contiki = join(with_malicious, 'contiki') contiki_rpl = join(contiki, 'core', 'net', 'rpl') with lcd(with_malicious): malicious = 'malicious.{}'.format(params["malicious_target"]) croot, csensor = 'root.{}'.format(params["target"]), 'sensor.{}'.format(params["target"]) # handle the malicious mote recompilation copy_folder(CONTIKI_FOLDER, with_malicious, includes=get_contiki_includes(params["malicious_target"])) if ext_lib is not None: remove_folder(contiki_rpl) copy_folder(ext_lib, contiki_rpl) apply_replacements(contiki_rpl, replacements) logger.debug(" > Making '{}'...".format(malicious)) stderr(local)("make malicious{} CONTIKI={}" .format(['', '.upload'][build], contiki), capture=True) if build: build = get_path(path, 'build', create=True) move_files(with_malicious, build, 'tmpimage.ihex') copy_files(with_malicious, build, malicious) move_files(with_malicious, without_malicious, malicious) local('make clean') remove_files(with_malicious, 'malicious.c') move_files(without_malicious, with_malicious, malicious) copy_files(without_malicious, with_malicious, croot, csensor) remove_folder(contiki)
def get_experiments(exp_file, silent=False): """ This function retrieves the dictionary of experiments with their parameters from a JSON campaign file. :param exp_file: input JSON simulation campaign file :param silent: specify whether an eventual error is to be raised or not :return: dictionary with the parsed experiments and their parameters """ if dirname(exp_file) == '': exp_file = join(EXPERIMENT_FOLDER, exp_file) exp_file = expanduser(exp_file) if not exp_file.endswith(".json"): exp_file += ".json" if not exists(exp_file): logger.critical("Simulation campaign JSON file does not exist !") logger.warning("Make sure you've generated a JSON simulation campaign file by using 'prepare' fabric command.") return return is_valid_commented_json(exp_file, return_json=True, logger=logger if not silent else None) or {}
def get_experiments(exp_file): """ This function retrieves the dictionary of experiments with their parameters from a JSON campaign file. :param exp_file: input JSON simulation campaign file :return: dictionary with the parsed experiments and their parameters """ if dirname(exp_file) == '': exp_file = join(EXPERIMENT_FOLDER, exp_file) exp_file = expanduser(exp_file) if not exp_file.endswith(".json"): exp_file += ".json" if not exists(exp_file): logger.critical("Simulation campaign JSON file does not exist !") logger.warning( "Make sure you've generated a JSON simulation campaign file by using 'prepare' fabric command." ) return with open(exp_file) as f: experiments = loads(jsmin(f.read())) return experiments
def wrapper(*args, **kwargs): """ This is the wrapper for the 'command' decorator, handling the following execution flow : - It first determines if this command is used with a console - In case of console, it performs a lexical analysis - Anyway, it performs a signature checking - It handles 'expand' parameter - It then handles 'exists' and 'not_exists' (in this order) - It finally selects the right behavior using 'behavior' """ # specific argument existence check through the 'not_exists' and 'exists' attributes def get_ask(): if attrs['on_boolean'] in kwargs.keys(): return kwargs[attrs['on_boolean']] try: param = sig.parameters[attrs['on_boolean']] try: return args[list(sig.parameters.values()).index(param)] except ( IndexError, ValueError ): # occurs if 'ask' was not specified in the arguments ; # in this case, take the default value from the signature return param.default except KeyError: # occurs if no 'ask' parameter is in the signature return False # log message formatting with specified arguments from 'args' or 'kwargs' def log_msg(lvl, msg): if kwargs.get('silent'): return if isinstance(msg, tuple): values = [] for a in msg[1:]: try: values.append(args[list( sig.parameters.keys()).index(a)]) except KeyError: value = kwargs.get(a) if value is not None: values.append(value) getattr(logger, lvl)(msg[0].format(*values)) else: getattr(logger, lvl)(msg) console = args[0] if len(args) > 0 and isinstance(args[0], Cmd) else None if console is not None: kwargs['console'] = console # lexical analysis if len(args) > 1 and console is not None: line = args[1] kwargs_tmp = {k: v for k, v in kwargs.items()} args, kwargs = lexer.analyze(line) if args is None and kwargs is None: print( console.badcmd_msg.format( "Invalid", '{} {}'.format(f.__name__, line))) return kwargs.update(kwargs_tmp) # bad signature check sig = signature(f) args = () if args == ( '', ) else args # occurs in case of Cmd ; an empty 'line' can be passed if a # command is called without any argument # - first, exclude variable arguments and keyword-arguments no_var_args = [ p for p in list(sig.parameters.values()) if not str(p).startswith('*') ] # - then, retrieve arguments without default values no_def_args = [p for p in no_var_args if p.default is not p.empty] # - now, if less input arguments are provided than the number of arguments without default value, # display an error if len(args) < len(no_var_args) - len(no_def_args): logger.critical("Bad input arguments !") logger.info( "This command has the following signature: {}{}".format( f.__name__, str(sig).replace(', **kwargs', ''))) logger.info( "Please check the documentation for more information.") return # expand a specified argument to a path if hasattr(f, 'expand'): arg, attrs = f.expand arg_idx = list(sig.parameters.keys()).index(arg) try: expanded = expanduser(join(attrs['into'], args[arg_idx])) except IndexError: # occurs when arg_idx out of range of args, meaning that the argument to be # expanded was not provided return # if an extension was provided and the file path does not end with it, just append it if attrs.get('ext') and not expanded.endswith("." + attrs['ext']): expanded += "." + attrs['ext'] # the expanded argument must not be saved to a new argument name, just replace its old value if attrs.get('new_arg') is None: args = tuple([ a if i != arg_idx else expanded for i, a in enumerate(args) ]) # otherwise, create the new argument if the name is not used yet elif attrs['new_arg'] not in list(sig.parameters.keys()): kwargs[attrs['new_arg']] = expanded if attrs.get( 'apply') is None else attrs['apply'](expanded) # when no 'path' kwarg is set, it must be added based on the direcotory name of the expanded arg if 'path' not in kwargs.keys(): kwargs['path'] = dirname(expanded) # if next commands require sudo, prompt now for privilege elevation if getattr(f, 'requires_sudo', False): system('sudo ls > /dev/null') # check for existence (or not) and ask for a confirmation to continue if required for fattr in ['exists', 'not_exists']: if hasattr(f, fattr): arg, attrs = getattr(f, fattr) try: arg_val = args[list(sig.parameters.keys()).index(arg)] except ValueError: arg_val = kwargs[arg] if (fattr == 'exists' and exists(arg_val)) or (fattr == 'not_exists' and not exists(arg_val)): if 'loglvl' in attrs.keys() and 'msg' in attrs.keys(): log_msg(attrs['loglvl'], attrs['msg']) if attrs.get('loglvl') in ('error', 'critical') or \ ((attrs.get('loglvl') in ('warning', 'info') or fattr == 'exists') and get_ask() and attrs.get('confirm') is not None and not std_input(attrs['confirm'], 'yellow') == 'yes'): return # run the command and catch exception if any if console is not None and len(args) > 0: console.clean_tasks() pending_tasks = { i['name']: str(o) for o, i in console.tasklist.items() if i['status'] == 'PENDING' } if args[0] not in pending_tasks.keys(): if hasattr(f, 'start_msg'): log_msg('info', f.start_msg) kwargs['task'] = f.__name__.lstrip('_') f.behavior( console, f.__base__ if f.behavior is MultiprocessedCommand else f, args[0], kwargs.get('path')).run(*args, **kwargs) else: logger.warning( "A task is still pending on this experiment ({})". format(pending_tasks[args[0]])) else: if hasattr(f, 'start_msg'): log_msg('info', f.start_msg) f(*args, **kwargs)
def wrapper(*args, **kwargs): """ This is the wrapper for the 'command' decorator, handling the following execution flow : - It first determines if this command is used with a console - In case of console, it performs a lexical analysis - Anyway, it performs a signature checking - It handles 'expand' parameter - It then handles 'exists' and 'not_exists' (in this order) - It finally selects the right behavior using 'behavior' """ # specific argument existence check through the 'not_exists' and 'exists' attributes def get_ask(): if attrs['on_boolean'] in kwargs.keys(): return kwargs[attrs['on_boolean']] try: param = sig.parameters[attrs['on_boolean']] try: return args[list(sig.parameters.values()).index(param)] except (IndexError, ValueError): # occurs if 'ask' was not specified in the arguments ; # in this case, take the default value from the signature return param.default except KeyError: # occurs if no 'ask' parameter is in the signature return False # log message formatting with specified arguments from 'args' or 'kwargs' def log_msg(lvl, msg): if kwargs.get('silent'): return if isinstance(msg, tuple): values = [] for a in msg[1:]: try: values.append(args[list(sig.parameters.keys()).index(a)]) except KeyError: value = kwargs.get(a) if value is not None: values.append(value) getattr(logger, lvl)(msg[0].format(*values)) else: getattr(logger, lvl)(msg) console = args[0] if len(args) > 0 and isinstance(args[0], Cmd) else None if console is not None: kwargs['console'] = console # lexical analysis if len(args) > 1 and console is not None: line = args[1] kwargs_tmp = {k: v for k, v in kwargs.items()} args, kwargs = lexer.analyze(line) if args is None and kwargs is None: print(console.badcmd_msg.format("Invalid", '{} {}'.format(f.__name__, line))) return kwargs.update(kwargs_tmp) # bad signature check sig = signature(f) args = () if args == ('',) else args # occurs in case of Cmd ; an empty 'line' can be passed if a # command is called without any argument # - first, exclude variable arguments and keyword-arguments no_var_args = [p for p in list(sig.parameters.values()) if not str(p).startswith('*')] # - then, retrieve arguments without default values no_def_args = [p for p in no_var_args if p.default is not p.empty] # - now, if less input arguments are provided than the number of arguments without default value, # display an error if len(args) < len(no_var_args) - len(no_def_args): logger.critical("Bad input arguments !") logger.info("This command has the following signature: {}{}" .format(f.__name__, str(sig).replace(', **kwargs', ''))) logger.info("Please check the documentation for more information.") return # expand a specified argument to a path if hasattr(f, 'expand'): arg, attrs = f.expand arg_idx = list(sig.parameters.keys()).index(arg) try: expanded = expanduser(join(attrs['into'], args[arg_idx])) except IndexError: # occurs when arg_idx out of range of args, meaning that the argument to be # expanded was not provided return if attrs.get('ext') and not expanded.endswith("." + attrs['ext']): expanded += "." + attrs['ext'] if attrs.get('new_arg') is None: args = tuple([a if i != arg_idx else expanded for i, a in enumerate(args)]) elif attrs['new_arg'] not in list(sig.parameters.keys()): kwargs[attrs['new_arg']] = expanded if attrs.get('apply') is None else attrs['apply'](expanded) # check for existence (or not) and ask for a confirmation to continue if required for fattr in ['exists', 'not_exists']: if hasattr(f, fattr): arg, attrs = getattr(f, fattr) try: arg_val = args[list(sig.parameters.keys()).index(arg)] except ValueError: arg_val = kwargs[arg] if (fattr == 'exists' and exists(arg_val)) or (fattr == 'not_exists' and not exists(arg_val)): if 'loglvl' in attrs.keys() and 'msg' in attrs.keys(): log_msg(attrs['loglvl'], attrs['msg']) if attrs.get('loglvl') in ('error', 'critical') or \ ((attrs.get('loglvl') in ('warning', 'info') or fattr == 'exists') and get_ask() and attrs.get('confirm') is not None and not std_input(attrs['confirm'], 'yellow') == 'yes'): return # run the command and catch exception if any if console is not None and len(args) > 0: console.clean_tasks() pending_tasks = {i['name']: str(o) for o, i in console.tasklist.items() if i['status'] == 'PENDING'} if args[0] not in pending_tasks.keys(): if hasattr(f, 'start_msg'): log_msg('info', f.start_msg) f.behavior(console, f.__base__ if f.behavior is MultiprocessedCommand else f, args[0]) \ .run(*args, **kwargs) else: logger.warning("A task is still pending on this experiment ({})".format(pending_tasks[args[0]])) else: if hasattr(f, 'start_msg'): log_msg('info', f.start_msg) f(*args, **kwargs)
def __make(name, ask=True, **kwargs): """ Make a new experiment. :param name: experiment name (or path to the experiment, if expanded in the 'command' decorator) :param ask: ask confirmation :param path: expanded path of the experiment (dynamically filled in through 'command' decorator with 'expand' :param kwargs: simulation keyword arguments (see the documentation for more information) """ global reuse_bin_path path = kwargs['path'] logger.debug(" > Validating parameters...") params = validated_parameters(kwargs) ext_lib = params.get("ext_lib") if ext_lib and not exists(ext_lib): logger.error("External library does not exist !") logger.critical("Make aborded.") return False logger.debug(" > Creating simulation...") # create experiment's directories check_structure(path, create=True, remove=True) templates = get_path(path, 'templates', create=True) get_path(templates, 'motes', create=True) # select the right malicious mote template and duplicate the simulation file copy_files((TEMPLATES_FOLDER, 'experiment'), templates, ('motes/{}.c'.format(params["mtype_root"]), 'motes/root.c'), ('motes/{}.c'.format(params["mtype_sensor"]), 'motes/sensor.c'), ('motes/{}.c'.format(params["mtype_malicious"]), 'motes/malicious.c'), 'motes/Makefile', 'Makefile', 'simulation.csc', 'script.js') # create experiment's files from templates then clean the templates folder replacements = render_templates(path, **params) remove_folder(templates) # now, write the config file without the list of motes del params['motes'] write_config(path, params) # now compile with settings(hide(*HIDDEN_ALL), warn_only=True): with_malicious = join(path, 'with-malicious', 'motes') without_malicious = join(path, 'without-malicious', 'motes') contiki = join(with_malicious, 'contiki') contiki_rpl = join(contiki, 'core', 'net', 'rpl') # copy a reduced version of Contiki where the debug flags can be set for RPL files set in DEBUG_FILES copy_folder(CONTIKI_FOLDER, with_malicious, includes=get_contiki_includes(params["target"], params["malicious_target"])) apply_debug_flags(contiki_rpl, debug=['NONE', 'PRINT'][params["debug"]]) with lcd(with_malicious): # first, compile root and sensor mote types croot, csensor = 'root.{}'.format(params["target"]), 'sensor.{}'.format(params["target"]) if reuse_bin_path is None or reuse_bin_path == with_malicious: logger.debug(" > Making '{}'...".format(croot)) stderr(local)("make root CONTIKI={}".format(contiki), capture=True) logger.debug(" > Making '{}'...".format(csensor)) stderr(local)("make sensor CONTIKI={}".format(contiki), capture=True) # here, files are moved ; otherwise, 'make clean' would also remove *.z1 move_files(with_malicious, without_malicious, croot, csensor) # after compiling, clean artifacts local('make clean') remove_files(with_malicious, 'root.c', 'sensor.c') else: copy_files(reuse_bin_path, without_malicious, croot, csensor) # second, handle the malicious mote compilation malicious = 'malicious.{}'.format(params["malicious_target"]) if ext_lib is not None: remove_folder(contiki_rpl) copy_folder(ext_lib, contiki_rpl) apply_replacements(contiki_rpl, replacements) logger.debug(" > Making '{}'...".format(malicious)) stderr(local)("make malicious CONTIKI={} TARGET={}" .format(contiki, params["malicious_target"]), capture=True) # temporary move compiled malicious mote, clean the compilation artifacts, move the malicious mote back # from the temporary location and copy compiled root and sensor motes move_files(with_malicious, without_malicious, malicious) local('make clean') move_files(without_malicious, with_malicious, malicious) copy_files(without_malicious, with_malicious, croot, csensor) # finally, remove compilation sources remove_files(with_malicious, 'malicious.c') remove_folder(contiki)
def __remake(name, build=False, **kwargs): """ Remake the malicious mote of an experiment. (meaning that it lets all simulation's files unchanged except ./motes/malicious.[target]) :param name: experiment name :param path: expanded path of the experiment (dynamically filled in through 'command' decorator with 'expand' """ set_logging(kwargs.get('loglevel')) path = kwargs['path'] logger.debug(" > Retrieving parameters...") params = read_config(path) ext_lib = params.get("ext_lib") if ext_lib and not exists(ext_lib): logger.error("External library does not exist !") logger.critical("Make aborted.") return False logger.debug(" > Recompiling malicious mote...") # remove former compiled malicious mote and prepare the template templates = get_path(path, 'templates', create=True) get_path(templates, 'motes', create=True) copy_files( (TEMPLATES_FOLDER, 'experiment'), templates, ('motes/{}.c'.format(params["mtype_malicious"]), 'motes/malicious.c')) # recreate malicious C file from template and clean the temporary template replacements = render_templates(path, only_malicious=True, **params) # then clean the temporary folder with templates remove_folder(templates) # now recompile with settings(hide(*HIDDEN_ALL), warn_only=True): with_malicious = join(path, 'with-malicious', 'motes') without_malicious = join(path, 'without-malicious', 'motes') contiki = join(with_malicious, split(CONTIKI_FOLDER)[-1]) contiki_rpl = join(contiki, 'core', 'net', 'rpl') with lcd(with_malicious): malicious = 'malicious.{}'.format(params["malicious_target"]) croot, csensor = 'root.{}'.format( params["target"]), 'sensor.{}'.format(params["target"]) # handle the malicious mote recompilation copy_folder(CONTIKI_FOLDER, with_malicious, includes=get_contiki_includes( params["malicious_target"])) if ext_lib is not None: remove_folder(contiki_rpl) copy_folder(ext_lib, contiki_rpl) apply_replacements(contiki_rpl, replacements) if build: logger.debug(" > Building '{}'...".format(malicious)) stderr(local)( "sudo make malicious.upload CONTIKI={} TARGET={}".format( contiki, params["malicious_target"])) build = get_path(path, 'build', create=True) move_files(with_malicious, build, 'tmpimage.ihex') copy_files(with_malicious, build, malicious) else: logger.debug(" > Making '{}'...".format(malicious)) stderr(local)("make malicious CONTIKI={} TARGET={}".format( contiki, params["malicious_target"]), capture=True) move_files(with_malicious, without_malicious, malicious) local('make clean') remove_files(with_malicious, 'malicious.c') move_files(without_malicious, with_malicious, malicious) copy_files(without_malicious, with_malicious, croot, csensor) remove_folder(contiki)
def __make(name, ask=True, **kwargs): """ Make a new experiment. :param name: experiment name (or path to the experiment, if expanded in the 'command' decorator) :param ask: ask confirmation :param path: expanded path of the experiment (dynamically filled in through 'command' decorator with 'expand' :param kwargs: simulation keyword arguments (see the documentation for more information) """ global reuse_bin_path set_logging(kwargs.get('loglevel')) path = kwargs['path'] logger.debug(" > Validating parameters...") params = validated_parameters(kwargs) ext_lib = params.get("ext_lib") if ext_lib and not exists(ext_lib): logger.error("External library does not exist !") logger.critical("Make aborded.") return False logger.debug(" > Creating simulation...") # create experiment's directories check_structure(path, create=True, remove=True) templates = get_path(path, 'templates', create=True) get_path(templates, 'motes', create=True) # select the right malicious mote template and duplicate the simulation file copy_files( (TEMPLATES_FOLDER, 'experiment'), templates, ('motes/{}.c'.format(params["mtype_root"]), 'motes/root.c'), ('motes/{}.c'.format(params["mtype_sensor"]), 'motes/sensor.c'), ('motes/{}.c'.format(params["mtype_malicious"]), 'motes/malicious.c'), 'motes/Makefile', 'Makefile', 'simulation.csc', 'script.js') # create experiment's files from templates then clean the templates folder replacements = render_templates(path, **params) remove_folder(templates) # now, write the config file without the list of motes del params['motes'] write_config(path, params) # now compile with settings(hide(*HIDDEN_ALL), warn_only=True): with_malicious = join(path, 'with-malicious', 'motes') without_malicious = join(path, 'without-malicious', 'motes') contiki = join(with_malicious, split(CONTIKI_FOLDER)[-1]) contiki_rpl = join(contiki, 'core', 'net', 'rpl') # copy a reduced version of Contiki where the debug flags can be set for RPL files set in DEBUG_FILES copy_folder(CONTIKI_FOLDER, with_malicious, includes=get_contiki_includes(params["target"], params["malicious_target"])) apply_debug_flags(contiki_rpl, debug=['NONE', 'PRINT'][params["debug"]]) with lcd(with_malicious): # first, compile root and sensor mote types croot, csensor = 'root.{}'.format( params["target"]), 'sensor.{}'.format(params["target"]) if reuse_bin_path is None or reuse_bin_path == with_malicious: logger.debug(" > Making '{}'...".format(croot)) stderr(local)("make root CONTIKI={}".format(contiki), capture=True) logger.debug(" > Making '{}'...".format(csensor)) stderr(local)("make sensor CONTIKI={}".format(contiki), capture=True) # here, files are moved ; otherwise, 'make clean' would also remove *.z1 move_files(with_malicious, without_malicious, croot, csensor) # after compiling, clean artifacts local('make clean') remove_files(with_malicious, 'root.c', 'sensor.c') else: copy_files(reuse_bin_path, without_malicious, croot, csensor) # second, handle the malicious mote compilation malicious = 'malicious.{}'.format(params["malicious_target"]) if ext_lib is not None: remove_folder(contiki_rpl) copy_folder(ext_lib, contiki_rpl) apply_replacements(contiki_rpl, replacements) logger.debug(" > Making '{}'...".format(malicious)) stderr(local)("make malicious CONTIKI={} TARGET={}".format( contiki, params["malicious_target"]), capture=True) # temporary move compiled malicious mote, clean the compilation artifacts, move the malicious mote back # from the temporary location and copy compiled root and sensor motes move_files(with_malicious, without_malicious, malicious) local('make clean') move_files(without_malicious, with_malicious, malicious) copy_files(without_malicious, with_malicious, croot, csensor) # finally, remove compilation sources remove_files(with_malicious, 'malicious.c') remove_folder(contiki)