def _execute(command): """ Execute the given command in the current directory, print error messages if the command failed. Args: command Command to execute Returns: Whether the operation is a success """ with tools.Context(None, "info"): sys.stdout.write("Executing " + repr(command) + "...") sys.stdout.flush() command = subprocess.run(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE) success = command.returncode == 0 with tools.Context(None, "info" if success else "warning"): if success: print(" done") else: print(" fail (" + str(command.returncode) + ")") for stdname in ("stdout", "stderr"): text = getattr(command, stdname).decode("utf8") if len(text) > 0: with tools.Context(stdname, "trace"): print(text) return success
def _loader(): """ Incrementally rebuild all libraries and register all local operations. """ try: # Check if the CUDA compiler is available nocuda = True for path in os.environ["PATH"].split(os.pathsep): if (pathlib.Path(path) / _build_cudabin).exists(): nocuda = False break # List all common headers headers = [] if _build_include is not None: for path in _build_include.iterdir(): if path.suffix in _build_exts_hdr: headers.append(path) # Compile libraries and load OP doneset = set() failset = set() for dirpath in pathlib.Path(__file__).resolve().parent.iterdir(): ident = dirpath.name[:3] if dirpath.is_dir() and ident in _loader_hooks.keys(): # Is a library directory if dirpath not in doneset and dirpath not in failset: so_path = _build_library(dirpath, doneset, failset, headers, nocuda=nocuda) loader = _loader_hooks[ident] if so_path is not None and loader is not None: # Successful build and loader needed loader(so_path) except Exception as err: with tools.Context(ident, "warning"): print("Loading failed while compiling " + repr(ident) + ": " + str(err)) with tools.Context("traceback", "trace"): traceback.print_exc()
def _loader_ctypes(so_path): """ Post-building ctypes loading operations. Args: so_path Shared object path """ try: lib = ctypes.CDLL(str(so_path)) register_py(so_path.stem[3:], lambda: lib) except Exception as err: with tools.Context(so_path.stem, "warning"): print("Loading failed for python interface " + repr(str(so_path)) + ": " + str(err)) with tools.Context("traceback", "trace"): traceback.print_exc()
def _run(topdir, name, seed, device, command): """ Run the attack experiments with the given named parameters. Args: topdir Parent result directory name Experiment unique name seed Experiment seed device Device on which to run the experiments command Command to run """ # Add seed to name name = "%s-%d" % (name, seed) # Process experiment with tools.Context(name, "info"): finaldir = topdir / name # Check whether the experiment was already successful if finaldir.exists(): tools.info("Experiment already processed.") return # Move-make the pending result directory resdir = move_directory(topdir / f"{name}.pending") resdir.mkdir(mode=0o755, parents=True) # Build the command args = command.build(seed, device, resdir) # Launch the experiment and write the standard output/error tools.trace((" ").join(shlex.quote(arg) for arg in args)) cmd_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if cmd_res.returncode == 0: tools.info("Experiment successful") else: tools.warning("Experiment failed") finaldir = topdir / f"{name}.failed" move_directory(finaldir) resdir.rename(finaldir) (finaldir / "stdout.log").write_bytes(cmd_res.stdout) (finaldir / "stderr.log").write_bytes(cmd_res.stderr)
def __init__(self, dataset, model, args): # Parse key:val arguments nbcores = len(os.sched_getaffinity(0)) if nbcores == 0: nbcores = 4 # Arbitrary fallback args = tools.parse_keyval(args, defaults={ "batch-size": 32, "eval-batch-size": 1024, "weight-decay": 0.00004, "label-smoothing": 0., "labels-offset": 0, "nb-fetcher-threads": nbcores, "nb-batcher-threads": nbcores }) if args["batch-size"] <= 0: raise tools.UserException( "Cannot make batches of non-positive size") # Report experiments with tools.Context("slim", None): print("Dataset name in use: " + repr(dataset[0]) + " (in " + repr(dataset[1]) + ")") print("Dataset preprocessing: " + (repr(args["preprocessing"]) if "preprocessing" in args else "<model default>")) print("Model name in use: " + repr(model)) # Finalization self.__args = args self.__dataset = dataset self.__preproc = args[ "preprocessing"] if "preprocessing" in args else model self.__model = model self.__cntr_wk = 0 # Worker instantiation counter self.__cntr_ev = 0 # Evaluator instantiation counter
def _loader_ops(so_path): """ Post-building custom ops loading operations. Args: so_path Shared object path """ try: lib = tf.load_op_library(str(so_path)) entries = lib.OP_LIST.ListFields()[0][1] try: while True: opname = entries.pop().ListFields()[0][1] opname = _camel_to_snake(opname) register_op(opname, getattr(lib, opname)) except IndexError: pass except Exception as err: with tools.Context(so_path.stem, "warning"): print("Loading failed for custom op " + repr(str(so_path)) + ": " + str(err)) with tools.Context("traceback", "trace"): traceback.print_exc()
def _run(name, seed, device, params): """ Run the attack experiments with the given named parameters. Args: name Experiment unique name seed Experiment seed device Device on which to run the experiments params Named parameters """ # Add seed to name name = "%s-%d" % (name, seed) # Process experiment with tools.Context(name, "info"): # Build and set the result directory result_dir = args.data_directory / name if result_dir.exists(): tools.info("Experiment already processed.") return result_dir.mkdir(mode=0o755, parents=True) # Add the missing options params["seed"] = str(seed) params["device"] = device params["result-directory"] = str(result_dir) # Launch the experiment and write the standard output/error def is_multi_param(param): return any(isinstance(param, typ) for typ in (list, tuple)) def param_to_str(param): if is_multi_param(param): return (" ").join(shlex.quote(str(val)) for val in param) return shlex.quote(str(param)) tools.trace("python3 -OO attack.py %s" % (" ").join("--%s %s" % (key, param_to_str(val)) for key, val in params.items())) command = ["python3", "-OO", "attack.py"] for key, val in params.items(): command.append("--%s" % (key, )) if is_multi_param(val): for subval in val: command.append(str(subval)) else: command.append(str(val)) cmd_res = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if cmd_res.returncode == 0: tools.info("Experiment successful") else: tools.warning("Experiment failed") (result_dir / "stdout.log").write_bytes(cmd_res.stdout) (result_dir / "stderr.log").write_bytes(cmd_res.stderr)
def _get_models(self): """ Lazy-initialize and return the map '__models'. Returns: '__models' """ # Fast-path already loaded if self.__models is not None: return self.__models # Initialize the dictionary self.__models = dict() # Populate this dictionary with TorchVision's models for name in dir(torchvision.models): if len(name) == 0 or name[0] == "_": # Ignore "protected" members continue builder = getattr(torchvision.models, name) if isinstance(builder, types.FunctionType): # Heuristic self.__models["torchvision-%s" % name.lower()] = builder # Dynamically add the custom models from subdirectory 'models/' def add_custom_models(name, module, _): nonlocal self # Check if has exports, fallback otherwise exports = getattr(module, "__all__", None) if exports is None: tools.warning("Model module %r does not provide '__all__'; falling back to '__dict__' for name discovery" % name) exports = (name for name in dir(module) if len(name) > 0 and name[0] != "_") # Register the association 'name -> constructor' for all the models exported = False for model in exports: # Check model name type if not isinstance(model, str): tools.warning("Model module %r exports non-string name %r; ignored" % (name, model)) continue # Recover instance from name constructor = getattr(module, model, None) # Check instance is callable (it's only an heuristic...) if not callable(constructor): continue # Register callable with composite name exported = True fullname = "%s-%s" % (name, model) if fullname in self.__models: tools.warning("Unable to make available model %r from module %r, as the name %r already exists" % (model, name, fullname)) continue self.__models[fullname] = constructor if not exported: tools.warning("Model module %r does not export any valid constructor name through '__all__'" % name) with tools.Context("models", None): tools.import_directory(pathlib.Path(__file__).parent / "models", {"__package__": "%s.models" % __package__}, post=add_custom_models) # Return the dictionary return self.__models
def __init__(self, args): # Parse key:val arguments args = tools.parse_keyval(args, defaults={"batch-size": 32}) if args["batch-size"] <= 0: raise tools.UserException("Cannot make batches of non-positive size") # Report loading with tools.Context("mnist", None): print("Loading MNIST dataset...") raw_data = tf.keras.datasets.mnist.load_data() # Finalization self.__args = args self.__raw_data = raw_data self.__datasets = None self.__cntr_wk = 0 # Worker instantiation counter self.__cntr_ev = 0 # Evaluator instantiation counter
def phishing(train=True, batch_size=None, root=None, download=False, *args, **kwargs): """ Phishing dataset generator builder. Args: train Whether to get the training slice of the dataset batch_size Batch size (None or 0 for all in one single batch) root Dataset cache root directory (None for default) download Whether to allow to download the dataset if not cached locally ... Ignored supplementary (keyword-)arguments Returns: Associated ataset generator """ with tools.Context("phishing", None): # Get the raw dataset inputs, labels = get_phishing(root or default_root, None if download is None else default_url_phishing) # Make and return the associated generator return experiments.batch_dataset(inputs, labels, train, batch_size, split=8400) # 8400 = 2⁴ × 3 × 5² × 7 (should help with divisibility)
def __call__(self, *args, **kwargs): try: ctx = tools.Context(hexchat.get_context()) return self.func(ctx, *args, **kwargs) except: tb = traceback.format_exc() if callable(self.on_error): try: retval = self.on_error(*args, **kwargs) except: tb = traceback.format_exc() else: retval = self.on_error errctx = get_error_context() errctx.prnt(tb) return retval
trace Whether to add trace prints for every important step of the computations Returns: List of loss tensors associated with 'device_models' """ raise NotImplementedError def accuracy(self, device_dataset, device_model, trace=False): """ Build an accuracy tensor on the specified devices, placement on parameter server by default. Args: device_dataset Dataset device name/function (same instance between calls if same task, i.e. can use 'is') device_models Model device names/functions, one per worker on the associated task trace Whether to add trace prints for every important step of the computations Returns: Map of metric string name -> aggregated metric tensor associated with 'device_models' """ raise NotImplementedError # ---------------------------------------------------------------------------- # # Experiment register and loader # Register instance _register = tools.ClassRegister("experiment") itemize = _register.itemize register = _register.register instantiate = _register.instantiate del _register # Load all local modules with tools.Context("experiments", None): tools.import_directory(pathlib.Path(__file__).parent, globals())
from .slim.preprocessing import preprocessing_factory from .slim.nets.nets_factory import networks_map # List available models models = list(networks_map.keys()) if len(models) == 0: raise tools.UserException("no model available in slim package") # List available datasets datasets = dict() dspath = pathlib.Path(__file__).parent / "datasets" if not dspath.is_dir(): raise tools.UserException("slim dataset at 'datasets' must be a directory") for path in dspath.iterdir(): if not tools.can_access(path, read=True): with tools.Context(None, "warning"): print("slim dataset " + repr(path.name + "/*") + " in 'datasets' is not read-able and has been ignored") continue if not path.is_dir(): # Must be after to first check for access rights... continue datasets[path.name] = str(path) if len(datasets) == 0: raise tools.UserException("no dataset available in slim package") # Register cross-product models-datasets for model in models: for dspair in datasets.items(): register("slim-" + model + "-" + dspair[0], SlimExperiment._make(dspair, model))
def _get_datasets(self): """ Lazy-initialize and return the map '__datasets'. Returns: '__datasets' """ global transforms # Fast-path already loaded if self.__datasets is not None: return self.__datasets # Initialize the dictionary self.__datasets = dict() # Populate this dictionary with TorchVision's datasets for name in dir(torchvision.datasets): if len(name) == 0 or name[0] == "_": # Ignore "protected" members continue constructor = getattr(torchvision.datasets, name) if isinstance(constructor, type): # Heuristic def make_builder(constructor, name): def builder(root, batch_size=None, shuffle=False, num_workers=1, *args, **kwargs): # Try to build the dataset instance data = constructor(root, *args, **kwargs) assert isinstance(data, torch.utils.data.Dataset), f"Internal heuristic failed: {name!r} was not a dataset name" # Ensure there is at least a tensor transformation for each torchvision dataset if name not in transforms: transforms[name] = torchvision.transforms.ToTensor() # Wrap into a loader batch_size = batch_size or len(data) loader = torch.utils.data.DataLoader(data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) # Wrap into an infinite batch sampler generator return make_sampler(loader) return builder self.__datasets[name.lower()] = make_builder(constructor, name) # Dynamically add the custom datasets from subdirectory 'datasets/' def add_custom_datasets(name, module, _): nonlocal self # Check if has exports, fallback otherwise exports = getattr(module, "__all__", None) if exports is None: tools.warning(f"Dataset module {name!r} does not provide '__all__'; falling back to '__dict__' for name discovery") exports = (name for name in dir(module) if len(name) > 0 and name[0] != "_") # Register the association 'name -> constructor' for all the datasets exported = False for dataset in exports: # Check dataset name type if not isinstance(dataset, str): tools.warning(f"Dataset module {name!r} exports non-string name {dataset!r}; ignored") continue # Recover instance from name constructor = getattr(module, dataset, None) # Check instance is callable (it's only an heuristic...) if not callable(constructor): continue # Register callable with composite name exported = True fullname = f"{name}-{dataset}" if fullname in self.__datasets: tools.warning(f"Unable to make available dataset {dataset!r} from module {name!r}, as the name {fullname!r} already exists") continue self.__datasets[fullname] = constructor if not exported: tools.warning(f"Dataset module {name!r} does not export any valid constructor name through '__all__'") with tools.Context("datasets", None): tools.import_directory(pathlib.Path(__file__).parent / "datasets", {"__package__": f"{__package__}.datasets"}, post=add_custom_datasets) # Return the dictionary return self.__datasets
# Return the selected function with the associated name return func def register(name, unchecked, check, upper_bound=None, influence=None): """ Simple registration-wrapper helper. Args: name GAR name unchecked Associated function (see module description) check Parameter validity check function upper_bound Compute the theoretical upper bound on the ratio non-Byzantine standard deviation / norm to use this aggregation rule: (n, f, d) -> float influence Attack acceptation ratio function """ global gars # Check if name already in use if name in gars: tools.warning("Unable to register %r GAR: name already in use" % name) return # Export the selected function with the associated name gars[name] = make_gar(unchecked, check, upper_bound=upper_bound, influence=influence) # Registered rules (mapping name -> aggregation rule) gars = dict() # Load all local modules with tools.Context("aggregators", None): tools.import_directory(pathlib.Path(__file__).parent, globals()) # Bind/overwrite the GAR name with the associated rules in globals() for name, rule in gars.items(): globals()[name] = rule
help= "Redirect the standard output to the given file (overwritten if exists), '-' for none, '-' by default" ) parser.add_argument( "--stderr-to", type=str, default="-", help= "Redirect the standard error output to the given file (overwritten if exists), '-' for none, '-' by default" ) parser.add_argument("--MPI", action="store_true", default=False, help="Run with MPI instead of gRPC.") with tools.Context("args", "info"): # Command line parsing args = parser.parse_args(sys.argv[1:]) # Early redirection handling if args.stdout_to != "-": path = pathlib.Path(args.stdout_to) sys.stdout = tools.MethodCallReplicator( sys.stdout, tools.ContextIOWrapper(path.open("w"), nocolor=True)) sys.stdout.write("Duplicating standard output to " + repr(str(path.resolve())) + os.linesep) if args.stderr_to != "-": path = pathlib.Path(args.stderr_to) sys.stderr = tools.MethodCallReplicator( sys.stderr, tools.ContextIOWrapper(path.open("w"), nocolor=True)) sys.stderr.write("Duplicating standard error output to " + repr(str(path.resolve())) + os.linesep)
def build_and_load_one(path, deps=[]): """ Check if the given directory is a module to build and load, and if yes recursively build and load its dependencies before it. Args: path Given directory path deps Dependent module paths Returns: True on success, False on failure, None if not a module """ nonlocal done_modules nonlocal fail_modules with tools.Context(path.name, "info"): ident = path.name[:3] if ident in ident_to_is_python.keys(): # Is a module directory if len(path.name) <= 3 or path.name[3] == "_": tools.warning("Skipped invalid module directory name " + repr(path.name)) return None if not path.exists(): tools.warning("Unable to build and load " + repr(str(path.name)) + ": module does not exist") fail_modules.append(path) # Mark as failed return False is_python_module = ident_to_is_python[ident] # Check if already built and loaded, or failed if path in done_modules: if len(deps) == 0 and debug_mode: tools.info("Already built and loaded " + repr(str(path.name))) return True if path in fail_modules: if len(deps) == 0: tools.warning("Was unable to build and load " + repr(str(path.name))) return False # Check for dependency cycle (disallowed as they may mess with the linker) if path in deps: tools.warning("Unable to build and load " + repr(str(path.name)) + ": dependency cycle found") fail_modules.append(path) # Mark as failed return False # Build and load dependencies this_ldflags = list(extra_ldflags) depsfile = path / dependencies_file if depsfile.exists(): for modname in depsfile.read_text().splitlines(): res = build_and_load_one(base_directory / modname, deps + [path]) if res == False: # Unable to build a dependency if len(deps) == 0: tools.warning("Unable to build and load " + repr(str(path.name)) + ": dependency " + repr(modname) + " build and load failed") fail_modules.append(path) # Mark as failed return False elif res == True: # Module and its sub-dependencies was/were built and loaded successfully this_ldflags.append("-Wl,--library=:" + str( (base_directory / modname / (modname + ".so")).resolve())) # List sources sources = [] for subpath in path.iterdir(): if subpath.is_file() and ("").join( subpath.suffixes) in source_suffixes: sources.append(str(subpath)) # Build and load this module try: res = torch.utils.cpp_extension.load( name=path.name, sources=sources, extra_cflags=extra_cflags, extra_cuda_cflags=extra_cuda_cflags, extra_ldflags=this_ldflags, extra_include_paths=extra_include_paths, build_directory=str(path), verbose=debug_mode, is_python_module=is_python_module) if is_python_module: glob[path.name[3:]] = res except Exception as err: tools.warning("Unable to build and load " + repr(str(path.name)) + ": " + str(err)) fail_modules.append(path) # Mark as failed return False done_modules.append(path) # Mark as built and loaded return True
def _build_and_load(): """ Incrementally rebuild all libraries and bind all local modules in the global. """ glob = globals() # Standard imports import os import pathlib import traceback import warnings # External imports import torch import torch.utils.cpp_extension # Internal imports import tools # Constants base_directory = pathlib.Path(__file__).parent.resolve() dependencies_file = ".deps" debug_mode_envname = "NATIVE_OPT" debug_mode_in_env = debug_mode_envname in os.environ if debug_mode_in_env: raw = os.environ[debug_mode_envname] value = raw.lower() if value in ["0", "n", "no", "false"]: debug_mode = True elif value in ["1", "y", "yes", "true"]: debug_mode = False else: tools.fatal( "%r defined in the environment, but with unexpected soft-boolean %r" % (debug_mode_envname, "%s=%s" % (debug_mode_envname, raw))) else: debug_mode = __debug__ cpp_std_envname = "NATIVE_STD" cpp_std = os.environ.get(cpp_std_envname, "c++14") ident_to_is_python = {"so_": False, "py_": True} source_suffixes = {".cpp", ".cc", ".C", ".cxx", ".c++"} extra_cflags = ["-Wall", "-Wextra", "-Wfatal-errors", "-std=%s" % cpp_std] if torch.cuda.is_available(): source_suffixes.update( set((".cu" + suffix) for suffix in source_suffixes)) source_suffixes.add(".cu") extra_cflags.append("-DTORCH_CUDA_AVAILABLE") extra_cuda_cflags = [ "-DTORCH_CUDA_AVAILABLE", "--expt-relaxed-constexpr", "-std=%s" % cpp_std ] extra_ldflags = ["-Wl,-L" + base_directory.root] extra_include_path = base_directory / "include" try: extra_include_paths = [str(extra_include_path.resolve())] except Exception: extra_include_paths = None warnings.warn("Not found include directory: " + repr(str(extra_include_path))) # Print configuration information cpp_std_message = "Native modules compiled with %s standard; (re)define %r in the environment to compile with another standard" % ( cpp_std, "%s=<standard>" % cpp_std_envname) if debug_mode: tools.warning(cpp_std_message) tools.warning( "Native modules compiled in debug mode; %sdefine %r in the environment or%s run python with -O/-OO options to compile in release mode" % ("re" if debug_mode_in_env else "", "%s=1" % debug_mode_envname, " undefine it and" if debug_mode_in_env else "")) extra_cflags += ["-O0", "-g"] else: quiet_envname = "NATIVE_QUIET" if quiet_envname not in os.environ: tools.trace(cpp_std_message) tools.trace( "Native modules compiled in release mode; %sdefine %r in the environment or%s run python without -O/-OO options to compile in debug mode" % ("re" if debug_mode_in_env else "", "%s=0" % debug_mode_envname, " undefine it and" if debug_mode_in_env else "")) tools.trace( "Define %r in the environment to hide these messages in release mode" % quiet_envname) extra_cflags += ["-O3", "-DNDEBUG"] # Variables done_modules = [] fail_modules = [] # Local procedures def build_and_load_one(path, deps=[]): """ Check if the given directory is a module to build and load, and if yes recursively build and load its dependencies before it. Args: path Given directory path deps Dependent module paths Returns: True on success, False on failure, None if not a module """ nonlocal done_modules nonlocal fail_modules with tools.Context(path.name, "info"): ident = path.name[:3] if ident in ident_to_is_python.keys(): # Is a module directory if len(path.name) <= 3 or path.name[3] == "_": tools.warning("Skipped invalid module directory name " + repr(path.name)) return None if not path.exists(): tools.warning("Unable to build and load " + repr(str(path.name)) + ": module does not exist") fail_modules.append(path) # Mark as failed return False is_python_module = ident_to_is_python[ident] # Check if already built and loaded, or failed if path in done_modules: if len(deps) == 0 and debug_mode: tools.info("Already built and loaded " + repr(str(path.name))) return True if path in fail_modules: if len(deps) == 0: tools.warning("Was unable to build and load " + repr(str(path.name))) return False # Check for dependency cycle (disallowed as they may mess with the linker) if path in deps: tools.warning("Unable to build and load " + repr(str(path.name)) + ": dependency cycle found") fail_modules.append(path) # Mark as failed return False # Build and load dependencies this_ldflags = list(extra_ldflags) depsfile = path / dependencies_file if depsfile.exists(): for modname in depsfile.read_text().splitlines(): res = build_and_load_one(base_directory / modname, deps + [path]) if res == False: # Unable to build a dependency if len(deps) == 0: tools.warning("Unable to build and load " + repr(str(path.name)) + ": dependency " + repr(modname) + " build and load failed") fail_modules.append(path) # Mark as failed return False elif res == True: # Module and its sub-dependencies was/were built and loaded successfully this_ldflags.append("-Wl,--library=:" + str( (base_directory / modname / (modname + ".so")).resolve())) # List sources sources = [] for subpath in path.iterdir(): if subpath.is_file() and ("").join( subpath.suffixes) in source_suffixes: sources.append(str(subpath)) # Build and load this module try: res = torch.utils.cpp_extension.load( name=path.name, sources=sources, extra_cflags=extra_cflags, extra_cuda_cflags=extra_cuda_cflags, extra_ldflags=this_ldflags, extra_include_paths=extra_include_paths, build_directory=str(path), verbose=debug_mode, is_python_module=is_python_module) if is_python_module: glob[path.name[3:]] = res except Exception as err: tools.warning("Unable to build and load " + repr(str(path.name)) + ": " + str(err)) fail_modules.append(path) # Mark as failed return False done_modules.append(path) # Mark as built and loaded return True # Main loop for path in base_directory.iterdir(): if path.is_dir(): try: build_and_load_one(path) except Exception as err: tools.warning("Exception while processing " + repr(str(path)) + ": " + str(err)) with tools.Context("traceback", "trace"): traceback.print_exc()
def _build_library(libpath, doneset, failset, headers, libstack=[], nocuda=False): """ (Re)build a library directory and its dependencies into their associated shared objects. Args: libpath Library directory path doneset Set of other, successfully built library directory paths to update failset Set of other, not compiling library directory paths to update headers List of shared header paths libstack Constant stack of dependent library directory paths nocuda CUDA compiler was not found, don't try to compile these files Returns: Built library shared object path (None on failure) """ with tools.Context(libpath.name, None): try: # Watch out for a dependency cycle libpath = libpath.resolve() hascycle = libpath in libstack libstack += [libpath] if hascycle: raise RuntimeError("dependency cycle found") # List dependencies and sources (per category) to build depends = [] # Library directory paths to build (some may already be built/not compile) shareds = [] # Shared object paths this library depends on headers = list(headers) # Header paths (initially copy of common headers) srcscpu = [] # C++ source paths srcsgpu = [] # CUDA source paths libroot = libpath.parent for path in libpath.iterdir(): try: path = path.resolve() except Exception: if path.is_symlink(): raise RuntimeError("missing dependency " + repr(os.readlink(str(path)))) continue # Else silently ignore file if path.is_dir(): if path.parent != libroot: # Silently ignore directory continue if _build_check_ident(path): # Is a valid dependency depends.append(path) else: if path.parent != libpath: # Silently ignore file continue exts = path.suffixes if len(exts) > 0: if exts[-1] in _build_exts_hdr: headers.append(path) continue elif exts[-1] in _build_exts_cuda: srcsgpu.append(path) continue elif exts[-1] in _build_exts_src: if len(exts) > 1 and exts[-2] in _build_exts_cuda: srcsgpu.append(path) else: srcscpu.append(path) continue elif exts[-1] in _build_exts_obj: continue tools.trace("Ignoring file " + repr(path.name) + ": no/unrecognized extension") if nocuda: # No CUDA compiler => we ignore any CUDA source srcsgpu.clear() # Process dependencies first for path in depends: if path in failset: raise RuntimeError("dependency " + repr(path.name) + " could not be built") if path in doneset: so_path = _build_so_path(path) else: so_path = _build_library(path, doneset, failset, headers, libstack, nocuda=nocuda) if so_path is None: raise RuntimeError("dependency " + repr(path.name) + " could not be built") shareds.append(so_path) # Process sources second obj_paths = [] # Object paths to link for src_path in srcscpu: obj_path = pathlib.Path(str(src_path) + ".o") if _build_must_rebuild(obj_path, headers + [src_path]): if not _execute(_build_cpp_cmd(src_path, obj_path, len(srcsgpu) > 0)): raise RuntimeError("C++ source " + repr(src_path.name) + " did not compile") obj_paths.append(obj_path) for src_path in srcsgpu: obj_path = pathlib.Path(str(src_path) + ".o") if _build_must_rebuild(obj_path, headers + [src_path]): if not _execute(_build_cuda_cmd(src_path, obj_path)): raise RuntimeError("CUDA source " + repr(src_path.name) + " did not compile") obj_paths.append(obj_path) # (Re)link the shared object so_path = _build_so_path(libpath) if _build_must_rebuild(so_path, obj_paths): if not _execute(_build_link_cmd(obj_paths, shareds, so_path)): raise RuntimeError("final shared object " + repr(so_path.name) + " could not be linked") doneset.add(libpath) return so_path except Exception as err: tools.warning("Build failed: " + str(err)) failset.add(libpath) return None
(name, message)) # Attack res = unchecked(f_real=f_real, **kwargs) # Forward asserted return value assert isinstance(res, list) and len( res ) == f_real, "Expected attack %r to return a list of %f Byzantine gradients, got %r" % ( name, f_real, res) return res # Select which function to call by default func = checked if __debug__ else unchecked # Bind all the (sub) functions to the selected function setattr(func, "check", check) setattr(func, "checked", checked) setattr(func, "unchecked", unchecked) # Export the selected function with the associated name attacks[name] = func # Registered attacks (mapping name -> attack) attacks = dict() # Load native and all local modules with tools.Context("attacks", None): tools.import_directory(pathlib.Path(__file__).parent, globals()) # Bind/overwrite the attack names with the associated attacks in globals() for name, attack in attacks.items(): globals()[name] = attack
default=0, help= "How many training steps between experiment checkpointing, 0 or leave '--result-directory' empty for no checkpointing" ) parser.add_argument( "--user-input-delta", type=int, default=0, help= "How many training steps between two prompts for user command inputs, 0 for no user input" ) # Parse command line return parser.parse_args(sys.argv[1:]) with tools.Context("cmdline", "info"): args = process_commandline() # Parse additional arguments for name in ("init_multi", "init_mono", "gar", "attack", "model", "loss", "criterion"): name = f"{name}_args" keyval = getattr(args, name) setattr(args, name, dict() if keyval is None else tools.parse_keyval(keyval)) # Count the number of real honest workers args.nb_honests = args.nb_workers - args.nb_real_byz if args.nb_honests < 0: tools.fatal( f"Invalid arguments: there are more real Byzantine workers ({args.nb_real_byz}) than total workers ({args.nb_workers})" ) # Check the learning rate and associated options
parser.add_argument("--result-directory", type=str, default=None, help="Path of the directory in which to save the experiment results (loss, cross-accuracy, ...) and checkpoints, empty for no saving") parser.add_argument("--evaluation-delta", type=int, default=100, help="How many training steps between model evaluations, 0 for no evaluation") parser.add_argument("--user-input-delta", type=int, default=0, help="How many training steps between two prompts for user command inputs, 0 for no user input") # Parse command line return parser.parse_args(sys.argv[1:]) with tools.Context("cmdline", "info"): args = process_commandline() # Parse additional arguments for name in ("gar", "attack", "model", "dataset", "loss", "criterion"): name = f"{name}_args" keyval = getattr(args, name) setattr(args, name, dict() if keyval is None else tools.parse_keyval(keyval)) # Count the number of real honest workers args.nb_honests = args.nb_workers - args.nb_real_byz if args.nb_honests < 0: tools.fatal(f"Invalid arguments: there are more real Byzantine workers ({args.nb_real_byz}) than total workers ({args.nb_workers})") # Check general training parameters if args.momentum < 0.: tools.fatal(f"Invalid arguments: negative momentum factor {args.momentum}") if args.dampening < 0.: tools.fatal(f"Invalid arguments: negative dampening factor {args.dampening}")
def interactive(glbs=None, lcls=None, prompt=">>> ", cprmpt="... "): """ Switch to a simple interactive prompt, execute CTRL+D (or equivalent) to leave. Args: glbs Globals dictionary to use, None to use caller's globals lcls Locals dictionary to use, None to use given globals or caller's locals/globals prompt Command prompt to display cprmpt Command prompt to display when continuing a line """ # Recover caller's globals and locals try: caller = sys._getframe().f_back except Exception: caller = None if glbs is None: tools.warning("Unable to recover caller's frame, locals and globals", context="interactive") if glbs is None: if caller is not None and hasattr(caller, "f_globals"): glbs = caller.f_globals else: glbs = dict() if lcls is None: if caller is not None and hasattr(caller, "f_locals"): lcls = caller.f_locals else: lcls = glbs # Command input and execution command = "" statement = False while True: print(prompt if len(command) == 0 else cprmpt, end="", flush=True) try: # Input new line try: line = input() print("\033[A") # Trick to "advertise" new line on stdout after new line on stdin except BaseException as err: if any(isinstance(err, cls) for cls in (EOFError, KeyboardInterrupt)): print() # Since no new line was printed by pressing ENTER return # Handle expression if not statement: try: res = eval(line, glbs, lcls) if res is not None: print(res) except SyntaxError: # Heuristic that we are dealing with a statement statement = True # Handle single or multi-line statement(s) if statement: if len(command) == 0: # Just went through trying an expression command = line try: exec(command, glbs, lcls) except SyntaxError: # Heuristic that we are dealing with a multi-line statement continue elif len(line) > 0: command += os.linesep + line continue else: # Multi-line statement is complete exec(command, glbs, lcls) except Exception: with tools.Context("uncaught", "error"): traceback.print_exc() command = "" statement = False
default="auto", help= "Comma-separated list of devices on which to run the experiments, used in a round-robin fashion" ) parser.add_argument( "--supercharge", type=int, default=1, help= "How many experiments are run in parallel per device, must be positive" ) # Parse command line return parser.parse_args(sys.argv[1:]) with tools.Context("cmdline", "info"): args = process_commandline() # Check the "supercharge" parameter if args.supercharge < 1: tools.fatal("Expected a positive supercharge value, got %d" % args.supercharge) # Make the result directories def check_make_dir(path): path = pathlib.Path(path) if path.exists(): if not path.is_dir(): tools.fatal("Given path %r must point to a directory" % (str(path), )) else: path.mkdir(mode=0o755, parents=True) return path
nocuda = True for path in os.environ["PATH"].split(os.pathsep): if (pathlib.Path(path) / _build_cudabin).exists(): nocuda = False break # List all common headers headers = [] if _build_include is not None: for path in _build_include.iterdir(): if path.suffix in _build_exts_hdr: headers.append(path) # Compile libraries and load OP doneset = set() failset = set() for dirpath in pathlib.Path(__file__).resolve().parent.iterdir(): ident = dirpath.name[:3] if dirpath.is_dir() and ident in _loader_hooks.keys(): # Is a library directory if dirpath not in doneset and dirpath not in failset: so_path = _build_library(dirpath, doneset, failset, headers, nocuda=nocuda) loader = _loader_hooks[ident] if so_path is not None and loader is not None: # Successful build and loader needed loader(so_path) except Exception as err: with tools.Context(ident, "warning"): print("Loading failed while compiling " + repr(ident) + ": " + str(err)) with tools.Context("traceback", "trace"): traceback.print_exc() with tools.Context("native", None): _loader()
default="auto", help= "Comma-separated list of devices on which to run the experiments, used in a round-robin fashion" ) parser.add_argument( "--supercharge", type=int, default=1, help= "How many experiments are run in parallel per device, must be positive" ) # Parse command line return parser.parse_args(sys.argv[1:]) with tools.Context("cmdline", "info"): args = process_commandline() # Check the "supercharge" parameter if args.supercharge < 1: tools.fatal( f"Expected a positive supercharge value, got {args.supercharge}") # Make the result directories def check_make_dir(path): path = pathlib.Path(path) if path.exists(): if not path.is_dir(): tools.fatal( f"Given path {str(path)!r} must point to a directory") else: path.mkdir(mode=0o755, parents=True) return path