def call_save(self, args: [str], prefix, postfix, extra_args: [str] = []) -> str: if self.parsed_display is None: raise ValueError("Called print when no display parser yet!") individuals_names = self.expand_to_ids(args) individuals = map( lambda id: self.parsed_display.get_individual(id), individuals_names) existing_individuals = self.get_existing_ids(individuals) # extra_args = self.command_line.get_non_save() for indv in existing_individuals: (print_cmd, filename) = self.command_generator.save_cmd([indv.unit_name], prefix, postfix)[0] trace( 3, 'printing ' + indv.id + "/" + indv.unit_name + " to " + filename) ex = self.execute(print_cmd) with io.open(filename, "w", encoding="latin-1") as fil: fil.write(ex.str_result + extra_args)
def _run(topdir, name, seed, device, command): """ Run the attack experiments with the given named parameters. Args: topdir Parent result directory name Experiment unique name seed Experiment seed device Device on which to run the experiments command Command to run """ # Add seed to name name = "%s-%d" % (name, seed) # Process experiment with tools.Context(name, "info"): finaldir = topdir / name # Check whether the experiment was already successful if finaldir.exists(): tools.info("Experiment already processed.") return # Move-make the pending result directory resdir = move_directory(topdir / f"{name}.pending") resdir.mkdir(mode=0o755, parents=True) # Build the command args = command.build(seed, device, resdir) # Launch the experiment and write the standard output/error tools.trace((" ").join(shlex.quote(arg) for arg in args)) cmd_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if cmd_res.returncode == 0: tools.info("Experiment successful") else: tools.warning("Experiment failed") finaldir = topdir / f"{name}.failed" move_directory(finaldir) resdir.rename(finaldir) (finaldir / "stdout.log").write_bytes(cmd_res.stdout) (finaldir / "stderr.log").write_bytes(cmd_res.stderr)
def run_save(self, save_arg, expected_file_count): folder = os.path.join(self.tempfolder, self._testMethodName) trace(5, u'Creating ', folder) os.mkdir(folder) prefix = os.path.join(folder, u'test_trace') Main([ u'fake_progname', u'-save', save_arg, u'-mxver', u'7', u'-prefix', prefix ], settings=self.settings).main() files = os.listdir(folder) trace( 3, u'Test ' + self._testMethodName + u" yielded " + unicode(len(files)), u" files:", files) if expected_file_count >= 0: self.assertEqual(expected_file_count, len(files), msg=u'Expected there to be ' + unicode(expected_file_count) + u" files after test. Got: " + u" ".join(files)) return folder
def __init__(self, argv:[str]): trace(0, argv) self.args = CommandLineParser(argv) self.mxver = int(self.args.get_arg('mxver', '0')) if self.mxver == 6: self.help_file = 'trace-help.output' self.display_file = 'trace-display.output' self.print_file = 'trace-print-6x.output' elif self.mxver == 7: self.help_file = 'trace-help-7.x.output' self.display_file = 'trace-display-7.x.output' self.print_file = 'trace-print-6x.output' # TODO, change to 7 else: raise ValueError("Unknown MX-One version " + str(self.mxver)) display_args = self.args.display print_args = self.args.print_args if self.args.help: self.print_help() elif not display_args is None: self.print_display() elif not print_args is None: self.print_print(print_args) else: print_str("args: " + " ".join(sys.argv[1:])) pass
def save_cmd( self, names: [str], prefix: str = "", postfix: str = ".log", ) -> [(str, str)]: def gen_tuple(unitname, id): cmd = CommandGenerator.get_cmd_print(id) filename = (prefix + sep + unitname + postfix).strip(sep) return (cmd, filename) if self.settings.file_separators.find(prefix[-1]) >= 0: sep = prefix[-1] prefix = prefix[:-1] elif self.settings.file_separators.find(postfix[0]) >= 0: sep = postfix[0] postfix = postfix[1:] else: sep = '_' res = [] for name in self.expand_names(names): id = self.display_output.get_id(name) if id is None: trace(2, "print_cmd: unknown unit " + name + ", not printed", file=sys.stderr) continue res.append(gen_tuple(name, id)) return res
def get_ids_of(self, name_or_list): res = [] for name in self.expand_names(name_or_list): ind = self.display_output.get_individual(name) if ind is None: trace(2, u"Unknown unit '" + name + u"', ignored") continue res.append(ind.id) return res
def get_auth_data(cls, server): ''' Получить данные аутентификации для конкретного сервера ''' tools.trace('Request authentication for: {0}'.format(server)) if server == '192.168.1.5': user = '******' password = '******' return cls(user, password) else: raise Exception('No auth data for server: {0}'.format(server))
def print_cmd(self, names): res = [] for name in names: id = self.display_output.get_id(name) if id is None: trace(2, u"print_cmd: unknown unit " + name + u", not printed", file=sys.stderr) continue cmd = CommandGenerator.get_cmd_print(id) res.append(cmd) return res
def test_save_b_indv(self): try: print u'==================================' print u'== test_print_b_indv' print u' temp folder: ' + self.tempfolder print u'==================================' folder = self.run_zip(u'SIPLP', 1) self.assertFileInOutput(folder, u'SIPLP') except Exception, e: trace(1, u'Test ', self._testMethodName, u' failed: ', unicode(e)) raise
def test_save_b_indv(self): try: print('==================================') print('== test_print_b_indv') print(' temp folder: ' + self.tempfolder) print('==================================') folder = self.run_zip('SIPLP', 1) self.assertFileInOutput(folder, 'SIPLP') except Exception as e: trace(1, 'Test ', self._testMethodName, ' failed: ', str(e)) raise
def set_textlevel(self, name, textlevel = u'normal'): res = [] for member in self.settings.expand_to_individuals(name): indv = self.display_output.get_individual(member) if indv is None: trace(2, u"Unknown gang-member '" + member + u"' Textlevel not changed") continue id = indv.id res.append(CommandGenerator.get_cmd_set_textlevel(id, textlevel)) return res
def _run(name, seed, device, params): """ Run the attack experiments with the given named parameters. Args: name Experiment unique name seed Experiment seed device Device on which to run the experiments params Named parameters """ # Add seed to name name = "%s-%d" % (name, seed) # Process experiment with tools.Context(name, "info"): # Build and set the result directory result_dir = args.data_directory / name if result_dir.exists(): tools.info("Experiment already processed.") return result_dir.mkdir(mode=0o755, parents=True) # Add the missing options params["seed"] = str(seed) params["device"] = device params["result-directory"] = str(result_dir) # Launch the experiment and write the standard output/error def is_multi_param(param): return any(isinstance(param, typ) for typ in (list, tuple)) def param_to_str(param): if is_multi_param(param): return (" ").join(shlex.quote(str(val)) for val in param) return shlex.quote(str(param)) tools.trace("python3 -OO attack.py %s" % (" ").join("--%s %s" % (key, param_to_str(val)) for key, val in params.items())) command = ["python3", "-OO", "attack.py"] for key, val in params.items(): command.append("--%s" % (key, )) if is_multi_param(val): for subval in val: command.append(str(subval)) else: command.append(str(val)) cmd_res = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if cmd_res.returncode == 0: tools.info("Experiment successful") else: tools.warning("Experiment failed") (result_dir / "stdout.log").write_bytes(cmd_res.stdout) (result_dir / "stderr.log").write_bytes(cmd_res.stderr)
def test_save_c_gang(self): try: print('==================================') print('== test_print_c_gang') print(' temp folder: ' + self.tempfolder) print('==================================') folder = self.run_zip('usual', 3) self.assertFileInOutput(folder, 'SIPLP') self.assertFileInOutput(folder, 'RMP') self.assertFileInOutput(folder, 'CMP') except Exception as e: trace(1, 'Test ', self._testMethodName, ' failed: ', str(e)) raise
def add_individuals(self, individuals, lim, textlevel, extra_args): trace(4, u"Adding individuals ", individuals) self.execute_all(self.command_generator.add_indv(individuals, lim, extra_args)) self.call_display() for id in individuals: indv = self.parsed_display.get_individual(id) if indv is None: trace(2, u'Failed to create induvidual ' + id) sys.exit(17) if indv.textlevel != textlevel: self.execute_all(self.command_generator.set_textlevel(id, textlevel))
def test_save_c_gang(self): try: print u'==================================' print u'== test_print_c_gang' print u' temp folder: ' + self.tempfolder print u'==================================' folder = self.run_zip(u'usual', 3) self.assertFileInOutput(folder, u'SIPLP') self.assertFileInOutput(folder, u'RMP') self.assertFileInOutput(folder, u'CMP') except Exception, e: trace(1, u'Test ', self._testMethodName, u' failed: ', unicode(e)) raise
def call_unknown_command(self): def find_gang(): for i, a in enumerate(self.command_line.argv[1:]): gang = self.expand_to_ids(a) if len(gang) > 0: pre = self.command_line.argv[:i] if i > 0 else [] post = self.command_line.argv[i+1:] if i+1 < len(self.command_line.argv) else [] return [ pre + [indv] + post for indv in gang ] return None gang_expanded = find_gang() or self.command_line.argv[1:] for a in gang_expanded: trace(4, a) self.execute(a)
def call_zip(self, args, zipfilename, prefix, postfix, extra_args=[]): if not isinstance(args, list): raise ValueError(u'Should be called with list of individuals') if self.parsed_display is None: raise ValueError(u"Called print when no display parser yet!") individuals_names = self.expand_to_ids(args) existing_individuals = self.get_existing_indivuduals(individuals_names) # extra_args = self.command_line.get_non_zip() with zipfile.ZipFile(zipfilename, u"w") as z: for indv in existing_individuals: (print_cmd, filename) = self.command_generator.save_cmd([indv.unit_name], prefix, postfix)[0] trace( 3, u'printing ' + indv.id + u"/" + indv.unit_name + u" to " + filename) ex = self.execute(print_cmd) z.writestr(filename, ex.result) trace(5, u"Wrote to " + zipfilename) try: trace(5, u"Size became " + unicode(os.path.getsize(zipfilename))) except: trace(1, u"Failed to save data to " + zipfilename)
def find_settings(file = None): ex = None files = [ file, os.path.join( os.path.dirname(os.path.realpath(__file__)), u'settings.json'), os.path.join( os.path.expanduser(u"~"), u'.mx-trace', u'settings.json'), os.path.join( os.path.expanduser(u"~"), u'.mx-trace.json') ] for f in files: if not f is None and os.path.exists(f): try: return Settings(f) except BaseException, ex: trace(3, u"Failed to open settings from " + f + u": " + unicode(ex))
def __exit__(self, *args, **kwargs): """ Exit context: stop chrono and print elapsed time. Args: ... Forwarded arguments """ # Measure elapsed time (in ns) elapsed = (time.time() - self._chrono) * 1000000000. # Print elapsed time for text in ["ns", "µs", "ms"]: if elapsed < 1000.: break elapsed /= 1000. else: text = "s" tools.trace("Execution time: %.3f %s" % (elapsed, text)) # Forward call super().__exit__(*args, **kwargs)
def __exit__(self, *args, **kwargs): """ Exit context: stop chrono and print elapsed time. Args: ... Forwarded arguments """ # Measure elapsed runtime (in ns) runtime = (time.time() - self._chrono) * 1000000000. # Recover ideal unit for unit in ("ns", "µs", "ms"): if runtime < 1000.: break runtime /= 1000. else: unit = "s" # Format and print string tools.trace(f"Execution time: {runtime:.3g} {unit}") # Forward call super().__exit__(*args, **kwargs)
def run_zip(self, save_arg: str, expected_file_count: int) -> str: folder = os.path.join(self.tempfolder, self._testMethodName) trace(5, 'Creating ', folder) os.mkdir(folder) m = Main([ 'fake_progname', '-zip', os.path.join(folder, "zipped"), save_arg, '-mxver', '7', '-prefix', 'trace_prefix' ], settings=self.settings) m.main() files = os.listdir(folder) trace(3, 'Test ' + self._testMethodName + " yielded " + str(len(files)), " files:", files) return folder
def run_zip(self, save_arg, expected_file_count): folder = os.path.join(self.tempfolder, self._testMethodName) trace(5, u'Creating ', folder) os.mkdir(folder) m = Main([ u'fake_progname', u'-zip', os.path.join(folder, u"zipped"), save_arg, u'-mxver', u'7', u'-prefix', u'trace_prefix' ], settings=self.settings) m.main() files = os.listdir(folder) trace( 3, u'Test ' + self._testMethodName + u" yielded " + unicode(len(files)), u" files:", files) return folder
def call_save(self, args, prefix, postfix, extra_args=[]): if self.parsed_display is None: raise ValueError(u"Called print when no display parser yet!") individuals_names = self.expand_to_ids(args) existing_individuals = [ self.parsed_display.get_individual(id) for id in individuals_names ] # extra_args = self.command_line.get_non_save() for indv in existing_individuals: (print_cmd, filename) = self.command_generator.save_cmd([indv.unit_name], prefix, postfix)[0] trace( 3, u'printing ' + indv.id + u"/" + indv.unit_name + u" to " + filename) ex = self.execute(print_cmd + extra_args) with io.open(filename, u"w", encoding=u"latin-1") as fil: fil.write(ex.str_result)
def find_settings(file: str = None) -> str: ex = None files = filter(None, [ file, os.path.join(os.path.dirname(os.path.realpath(__file__)), 'settings.json'), os.path.join(os.path.expanduser("~"), '.mx-trace', 'settings.json'), os.path.join(os.path.expanduser("~"), '.mx-trace.json') ]) for f in files: if os.path.exists(f): try: return Settings(f) except BaseException as ex: trace(3, "Failed to open settings from " + f + ": " + str(ex)) trace( 3, "Failed to open settings from any known file " + "\n " + "\n ".join(files)) return Settings("{\n}") # Return empty settings
def __init__(self, program, args, trace_cmd_level = 7, trace_result_level = 8): self.program = program self.args = args prog_args = [self.program] if not args is None: prog_args += args trace(trace_cmd_level, u'Executor ' + program + u' ', args) try: self.result = subprocess.check_output(prog_args) if tracelevel >= trace_result_level: trace(trace_result_level, self.str_result) except subprocess.CalledProcessError, e: trace(1, e.cmd, u'failed with ', e.returncode) trace(1, e.output) sys.exit(e.returncode)
def _build_library(libpath, doneset, failset, headers, libstack=[], nocuda=False): """ (Re)build a library directory and its dependencies into their associated shared objects. Args: libpath Library directory path doneset Set of other, successfully built library directory paths to update failset Set of other, not compiling library directory paths to update headers List of shared header paths libstack Constant stack of dependent library directory paths nocuda CUDA compiler was not found, don't try to compile these files Returns: Built library shared object path (None on failure) """ with tools.Context(libpath.name, None): try: # Watch out for a dependency cycle libpath = libpath.resolve() hascycle = libpath in libstack libstack += [libpath] if hascycle: raise RuntimeError("dependency cycle found") # List dependencies and sources (per category) to build depends = [] # Library directory paths to build (some may already be built/not compile) shareds = [] # Shared object paths this library depends on headers = list(headers) # Header paths (initially copy of common headers) srcscpu = [] # C++ source paths srcsgpu = [] # CUDA source paths libroot = libpath.parent for path in libpath.iterdir(): try: path = path.resolve() except Exception: if path.is_symlink(): raise RuntimeError("missing dependency " + repr(os.readlink(str(path)))) continue # Else silently ignore file if path.is_dir(): if path.parent != libroot: # Silently ignore directory continue if _build_check_ident(path): # Is a valid dependency depends.append(path) else: if path.parent != libpath: # Silently ignore file continue exts = path.suffixes if len(exts) > 0: if exts[-1] in _build_exts_hdr: headers.append(path) continue elif exts[-1] in _build_exts_cuda: srcsgpu.append(path) continue elif exts[-1] in _build_exts_src: if len(exts) > 1 and exts[-2] in _build_exts_cuda: srcsgpu.append(path) else: srcscpu.append(path) continue elif exts[-1] in _build_exts_obj: continue tools.trace("Ignoring file " + repr(path.name) + ": no/unrecognized extension") if nocuda: # No CUDA compiler => we ignore any CUDA source srcsgpu.clear() # Process dependencies first for path in depends: if path in failset: raise RuntimeError("dependency " + repr(path.name) + " could not be built") if path in doneset: so_path = _build_so_path(path) else: so_path = _build_library(path, doneset, failset, headers, libstack, nocuda=nocuda) if so_path is None: raise RuntimeError("dependency " + repr(path.name) + " could not be built") shareds.append(so_path) # Process sources second obj_paths = [] # Object paths to link for src_path in srcscpu: obj_path = pathlib.Path(str(src_path) + ".o") if _build_must_rebuild(obj_path, headers + [src_path]): if not _execute(_build_cpp_cmd(src_path, obj_path, len(srcsgpu) > 0)): raise RuntimeError("C++ source " + repr(src_path.name) + " did not compile") obj_paths.append(obj_path) for src_path in srcsgpu: obj_path = pathlib.Path(str(src_path) + ".o") if _build_must_rebuild(obj_path, headers + [src_path]): if not _execute(_build_cuda_cmd(src_path, obj_path)): raise RuntimeError("CUDA source " + repr(src_path.name) + " did not compile") obj_paths.append(obj_path) # (Re)link the shared object so_path = _build_so_path(libpath) if _build_must_rebuild(so_path, obj_paths): if not _execute(_build_link_cmd(obj_paths, shareds, so_path)): raise RuntimeError("final shared object " + repr(so_path.name) + " could not be linked") doneset.add(libpath) return so_path except Exception as err: tools.warning("Build failed: " + str(err)) failset.add(libpath) return None
def trace(self, lvl, *argv): argv = ("TestCommendLineParser: ", ) + argv return tools.trace(lvl, argv)
def trace(self, lvl, *argv): argv = (u"TestCommendGenerator: ", ) + argv return tools.trace(lvl, argv)
def main(self): trace(7, u"main method: " + self.command_line.program_name + u" args: [ " + u", ".join(self.command_line.original_args)+ u" ]") display_settings = self.command_line.display_settings if display_settings: return self.call_display_settings() if self.command_line.gangs_request: return self.call_display_gangs() help_args = self.command_line.help if help_args: return self.call_help() display_args = self.command_line.display if not display_args is None: trace(3, u"display " + u" ".join(display_args), file=sys.stderr) self.call_display(display_args + self.command_line.display_extra_args().argv) print self.parsed_display return add_args = self.command_line.add if not add_args is None: trace(3, u"add " + u" ".join(add_args)) self.call_display() add_extra_args = self.command_line.add_extra_args().argv self.call_add(add_args.split(u','), self.command_line.lim, self.command_line.textlevel or self.settings.default_textlevel, add_extra_args) return remove_args = self.command_line.remove if not remove_args is None: trace(3, u"remove " + u" ".join(remove_args)) self.call_display() remove_extra_args = self.command_line.remove_extra_args().argv self.call_remove(remove_args.split(u','), remove_extra_args) return start_args = self.command_line.start if not start_args is None: trace(3, u"start " + u" ".join(start_args)) self.call_display() start_extra_args = self.command_line.start_extra_args().argv self.call_start(start_args.split(u','), self.command_line.lim, self.command_line.textlevel or self.settings.default_textlevel, start_extra_args) return stop_args = self.command_line.stop if not stop_args is None: trace(3, u"stop " + u" ".join(stop_args)) self.call_display() self.call_stop(stop_args.split(u','), self.command_line.stop_extra_args().argv) return print_args = self.command_line.print_args if not print_args is None: trace(3, u"print " + print_args) self.call_display() extra_args = self.command_line.print_extra_args().argv printout = self.call_print([print_args], extra_args) print printout return printout save_args = self.command_line.save if not save_args is None: prefix=self.command_line.file_prefix or self.settings.file_prefix postfix=self.command_line.file_postfix or self.settings.file_postfix trace(3, u"save " + save_args , u", prefix=" , prefix , u", postfix=" , postfix) self.call_display() self.call_save(save_args.split(u','), prefix, postfix) return zip_args = self.command_line.zip if not zip_args is None: prefix = self.command_line.file_prefix or self.settings.zip_prefix postfix = self.command_line.file_postfix or self.settings.zip_postfix zip_file = zip_args[0] _ignored , file_extension = os.path.splitext(zip_file) if len(file_extension) == 0: zip_file = zip_file.rstrip(u'.') + u'.zip' individuals = zip_args[1:] trace(3, u"zip ", individuals, u" to ", zip_file , u", prefix=" , prefix , u", postfix=" , postfix) self.call_display() self.call_zip(individuals, zip_file, prefix, postfix) return trace(2, u'Unknown command, calling trace verbatim for all individuals') self.call_display() self.call_unknown_command() return
if len(args) < 1: usage(argv[0], 1) for applicationFile in args: if not os.path.exists(applicationFile): print >> sys.stderr, "Application file '%s' does not exists!" % applicationFile continue applicationName = applicationFile[0:-4] filePath = os.path.join (os.path.realpath("."), applicationFile) importMsg = "Importing application '%s' from file '%s'" % (applicationName, filePath) if repository.exists(applicationName): print ("Application '%s' already exists - skipped" % applicationName) elif dryRun: print (importMsg + " - skipped (DryRun!)") else: print (importMsg) try: deployit.importPackage(filePath) # Check out the TODO in the import section! # except ImporterException, e: # print >> sys.stderr, "Could not import DAR from '%s': '%s'" % (applicationFile, e.strerror) except: print >> sys.stderr, "Could not import DAR from '%s'!" % (applicationFile) if not skipError: sys.exit(2) # The usual way to start main does not work, obviously the script is embedded somehow? # if __name__ == "__main__": trace ("Calling main(" + str(sys.argv) + ")") main(sys.argv)
class Main(object): help_str = \ u""" trace-helper, can call and group calls to MX-trace. If starting something not yet defined, it is auto-defined. Possible to start/stop/print on name, not only ID. Possible to configure groups if units, all listening to the same name Switches not used by this program should be passed down to trace Many settings have default values, change ~/.mx-trace.json -display_settings to output current settings-file -gangs Prints which gangs are defined -add indivuduals_or_gang (same as -unit in trace) -remove indivuduals_or_gang -start indivuduals_or_gang -stop indivuduals_or_gang -clear indivuduals_or_gang -print indivuduals_or_gang -save indivudual_or_gang -prefix trace -postfix log Print all the individuals to separate files, with specified prefix/postfix. (Prefix might include paths) -zip output_file.zip indivudual_or_gang Like save, but generate 1 zip file with all traces If specifying many individuals/gangs, separate with space """ def __init__(self, program_name, argv, settings = None): self.command_line = CommandLineParser(program_name, argv) self.settings = settings or Main.find_settings(self.command_line.settings_file) self.parsed_display = ParseDisplayOutput(u"") self.command_generator = CommandGenerator(self.parsed_display, self.settings) def set_parsed_display(self, val): self.parsed_display = val self.command_generator = CommandGenerator(val, self.settings) def execute(self, args): return Executor(self.settings.trace_cmd, self.settings.trace_args + args, trace_cmd_level=self.settings.debug_trace_commands) def execute_all(self, list_of_args): if not isinstance(list_of_args, list): raise ValueError(u"args should be list of command-line-lists, was just " + unicode(type(list_of_args))) if not isinstance(list_of_args[0], list): raise ValueError(u"args should be list of command-line-lists, was just " + unicode(type(list_of_args))) return u"\n".join([self.execute(arg).str_result for arg in list_of_args]) @staticmethod def find_settings(file = None): ex = None files = [ file, os.path.join( os.path.dirname(os.path.realpath(__file__)), u'settings.json'), os.path.join( os.path.expanduser(u"~"), u'.mx-trace', u'settings.json'), os.path.join( os.path.expanduser(u"~"), u'.mx-trace.json') ] for f in files: if not f is None and os.path.exists(f): try: return Settings(f) except BaseException, ex: trace(3, u"Failed to open settings from " + f + u": " + unicode(ex)) trace(3, u"Failed to open settings from any known file " + u"\n " + u"\n ".join(files) ) return Settings(u"{\n}") # Return empty settings