def test_in_toto_run_compare_with_and_without_metadata_directory(self): """Successfully run with and without metadata directory, compare the signed is equal""" tmp_dir = os.path.realpath(tempfile.mkdtemp(dir=os.getcwd())) in_toto_run(self.step_name, [self.test_artifact], [self.test_artifact], ["python", "--version"], True, self.key, metadata_directory=tmp_dir) file_path = os.path.join(tmp_dir, FILENAME_FORMAT.format(step_name=self.step_name, keyid=self.key["keyid"])) link_dump_with_md = Metablock.load(file_path) in_toto_run(self.step_name, [self.test_artifact], [self.test_artifact], ["python", "--version"], True, self.key) link_dump_without_md = Metablock.load( FILENAME_FORMAT.format(step_name=self.step_name, keyid=self.key["keyid"])) self.assertEqual(repr(link_dump_with_md.signed), repr(link_dump_without_md.signed))
def setUpClass(self): """Create and change into temporary directory, generate key pair, dummy artifact and base arguments. """ self.working_dir = os.getcwd() self.test_dir = tempfile.mkdtemp() # Copy gpg keyring self.default_gpg_keyid = "8465a1e2e0fb2b40adb2478e18fb3f537e0c8a17" self.default_gpg_subkeyid = "c5a0abe6ec19d0d65f85e2c39be9df5131d924e9" self.non_default_gpg_keyid = "8288ef560ed3795f9df2c0db56193089b285da58" gpg_keyring_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "gpg_keyrings", "rsa") self.gnupg_home = os.path.join(self.test_dir, "rsa") shutil.copytree(gpg_keyring_path, self.gnupg_home) os.chdir(self.test_dir) self.key_path = "test_key" generate_and_write_rsa_keypair(self.key_path) self.key = prompt_import_rsa_key_from_file(self.key_path) self.test_step = "test_step" self.test_link = FILENAME_FORMAT.format(step_name=self.test_step, keyid=self.key["keyid"]) self.test_artifact = "test_artifact" open(self.test_artifact, "w").close()
def test_in_toto_run_compare_dumped_with_returned_link(self): """Successfully run, compare dumped link is equal to returned link. """ link = in_toto_run(self.step_name, [self.test_artifact], [self.test_artifact], ["python", "--version"], True, self.key) link_dump = Metablock.load( FILENAME_FORMAT.format(step_name=self.step_name, keyid=self.key["keyid"])) self.assertEqual(repr(link), repr(link_dump))
def tearDown(self): """Remove link file if it was created. """ try: os.remove( FILENAME_FORMAT.format(step_name=self.step_name, keyid=self.key["keyid"])) except OSError: pass
def in_toto_record_stop(step_name, key, product_list): """ <Purpose> Finishes creating link metadata for a multi-part in-toto step. Loads signing key and unfinished link metadata file from disk, verifies that the file was signed with the key, records products, updates unfinished Link object (products and signature), removes unfinished link file from and stores new link file to disk. <Arguments> step_name: A unique name to relate link metadata with a step defined in the layout. key: Private key to sign link metadata. Format is securesystemslib.formats.KEY_SCHEMA product_list: List of file or directory paths that should be recorded as products. <Exceptions> None. <Side Effects> Writes newly created link metadata file to disk using the filename scheme from link.FILENAME_FORMAT Removes unfinished link file link.UNFINISHED_FILENAME_FORMAT from disk <Returns> None. """ fn = FILENAME_FORMAT.format(step_name=step_name, keyid=key["keyid"]) unfinished_fn = UNFINISHED_FILENAME_FORMAT.format(step_name=step_name, keyid=key["keyid"]) log.info("Stop recording '{}'...".format(step_name)) # Expects an a file with name UNFINISHED_FILENAME_FORMAT in the current dir log.info("Loading preliminary link metadata '{}'...".format(unfinished_fn)) link_metadata = Metablock.load(unfinished_fn) # The file must have been signed by the same key log.info("Verifying preliminary link signature...") keydict = {key["keyid"]: key} link_metadata.verify_signatures(keydict) if product_list: log.info("Recording products '{}'...".format(", ".join(product_list))) link_metadata.signed.products = record_artifacts_as_dict(product_list) log.info("Updating signature with key '{:.8}...'...".format(key["keyid"])) link_metadata.signatures = [] link_metadata.sign(key) log.info("Storing link metadata to '{}'...".format(fn)) link_metadata.dump(fn) log.info("Removing unfinished link metadata '{}'...".format(unfinished_fn)) os.remove(unfinished_fn)
def load_links_for_layout(layout): """ <Purpose> Try to load all existing metadata files for each Step of the Layout from the current directory. For each step the metadata might consist of multiple (thresholds) Link or Layout (sub-layouts) files. <Arguments> layout: Layout object <Side Effects> Calls function to read files from disk <Returns> A dictionary carrying all the found metadata corresponding to the passed layout, e.g.: { <step name> : { <functionary key id> : <Metablock containing a Link or Layout object>, ... }, ... } """ steps_metadata = {} # Iterate over all the steps in the layout for step in layout.steps: links_per_step = {} # We try to load a link for every authorized functionary, but don't fail # if the file does not exist (authorized != required) # FIXME: Should we really pass on IOError, or just skip inexistent links for keyid in step.pubkeys: filename = FILENAME_FORMAT.format(step_name=step.name, keyid=keyid) try: metadata = Metablock.load(filename) links_per_step[keyid] = metadata except IOError as e: pass # Check if the step has been performed by enough number of functionaries if len(links_per_step) < step.threshold: raise in_toto.exceptions.LinkNotFoundError( "Step '{0}' requires '{1}'" " link metadata file(s), found '{2}'.".format( step.name, step.threshold, len(links_per_step))) steps_metadata[step.name] = links_per_step return steps_metadata
def setUpClass(self): """Create and change into temporary directory, generate key pair, dummy artifact and base arguments. """ self.set_up_test_dir() self.set_up_gpg_keys() self.set_up_keys() self.test_step = "test_step" self.test_link_rsa = FILENAME_FORMAT.format(step_name=self.test_step, keyid=self.rsa_key_id) self.test_link_ed25519 = FILENAME_FORMAT.format( step_name=self.test_step, keyid=self.ed25519_key_id) self.test_link_rsa_enc = FILENAME_FORMAT.format( step_name=self.test_step, keyid=self.rsa_key_enc_id) self.test_link_ed25519_enc = FILENAME_FORMAT.format( step_name=self.test_step, keyid=self.ed25519_key_enc_id) self.test_artifact = "test_artifact" open(self.test_artifact, "w").close()
def test_in_toto_run_with_metadata_directory(self): """Successfully run with metadata directory, compare dumped link is equal to returned link""" tmp_dir = os.path.realpath(tempfile.mkdtemp(dir=os.getcwd())) link = in_toto_run(self.step_name, [self.test_artifact], [self.test_artifact], ["python", "--version"], True, self.key, metadata_directory=tmp_dir) file_path = os.path.join(tmp_dir, FILENAME_FORMAT.format(step_name=self.step_name, keyid=self.key["keyid"])) link_dump = Metablock.load(file_path) self.assertEqual(repr(link), repr(link_dump))
def test_main_with_specified_gpg_key(self): """Test CLI command with specified gpg key. """ args = [ "-n", self.test_step, "--gpg", self.non_default_gpg_keyid, "--gpg-home", self.gnupg_home, "--", "ls" ] self.assert_cli_sys_exit(args, 0) link_filename = FILENAME_FORMAT.format( step_name=self.test_step, keyid=self.non_default_gpg_keyid) self.assertTrue(os.path.exists(link_filename))
def test_main_with_default_gpg_key(self): """Test CLI command with default gpg key. """ args = [ "-n", self.test_step, "--gpg", "--gpg-home", self.gnupg_home, "--", "python", "--version" ] self.assert_cli_sys_exit(args, 0) link_filename = FILENAME_FORMAT.format(step_name=self.test_step, keyid=self.default_gpg_subkeyid) self.assertTrue(os.path.exists(link_filename))
def setUpClass(self): """Create and change into temporary directory, generate key pair, dummy artifact and base arguments. """ self.set_up_test_dir() self.set_up_gpg_keys() self.rsa_key_path = "test_key_rsa" generate_and_write_rsa_keypair(self.rsa_key_path) self.rsa_key = import_private_key_from_file(self.rsa_key_path, KEY_TYPE_RSA) self.ed25519_key_path = "test_key_ed25519" generate_and_write_ed25519_keypair(self.ed25519_key_path) self.ed25519_key = import_private_key_from_file( self.ed25519_key_path, KEY_TYPE_ED25519) self.test_step = "test_step" self.test_link_rsa = FILENAME_FORMAT.format( step_name=self.test_step, keyid=self.rsa_key["keyid"]) self.test_link_ed25519 = FILENAME_FORMAT.format( step_name=self.test_step, keyid=self.ed25519_key["keyid"]) self.test_artifact = "test_artifact" open(self.test_artifact, "w").close()
def _sign_and_dump_metadata(metadata, args): """ <Purpose> Internal method to sign link or layout metadata and dump it to disk. <Arguments> metadata: Metablock object (contains Link or Layout object) args: see argparser <Exceptions> SystemExit(0) if signing is successful SystemExit(2) if any exception occurs """ try: if not args.append: metadata.signatures = [] for key_path in args.key: key = util.prompt_import_rsa_key_from_file(key_path) metadata.sign(key) # Only relevant when signing Link metadata, where there is only one key keyid = key["keyid"] if args.output: out_path = args.output elif metadata._type == "link": out_path = FILENAME_FORMAT.format(step_name=metadata.signed.name, keyid=keyid) elif metadata._type == "layout": out_path = args.file log.info("Dumping {0} to '{1}'...".format(metadata._type, out_path)) metadata.dump(out_path) sys.exit(0) except Exception as e: log.error("The following error occurred while signing: " "{}".format(e)) sys.exit(2)
def test_main_with_encrypted_ed25519_key(self): """Test CLI command with encrypted ed25519 key. """ key_path = "test_key_ed25519_enc" password = "******" generate_and_write_ed25519_keypair(key_path, password) args = [ "-n", self.test_step, "--key", key_path, "--key-type", "ed25519", "--", "ls" ] with mock.patch('in_toto.util.prompt_password', return_value=password): key = import_private_key_from_file(key_path, KEY_TYPE_ED25519) linkpath = FILENAME_FORMAT.format(step_name=self.test_step, keyid=key["keyid"]) self.assert_cli_sys_exit(args, 0) self.assertTrue(os.path.exists(linkpath))
def setUpClass(self): """Create and change into temporary directory, generate key pair, dummy artifact and base arguments. """ self.working_dir = os.getcwd() self.test_dir = tempfile.mkdtemp() os.chdir(self.test_dir) self.key_path = "test_key" generate_and_write_rsa_keypair(self.key_path) self.key = prompt_import_rsa_key_from_file(self.key_path) self.test_step = "test_step" self.test_link = FILENAME_FORMAT.format(step_name=self.test_step, keyid=self.key["keyid"]) self.test_artifact = "test_artifact" open(self.test_artifact, "w").close()
def setUpClass(self): """Creates and changes into temporary directory and prepares two layouts. The superlayout, which has one step and its sublayout, which is the usual demo layout (write code, package, inspect tar). """ # Backup original cwd self.working_dir = os.getcwd() # Find demo files demo_files = os.path.join(os.path.dirname(os.path.realpath(__file__)), "demo_files") # Create and change into temporary directory self.test_dir = os.path.realpath(tempfile.mkdtemp()) os.chdir(self.test_dir) # Copy demo files to temp dir for file in os.listdir(demo_files): shutil.copy(os.path.join(demo_files, file), self.test_dir) # Import sub layout signing (private) and verifying (public) keys alice = import_rsa_key_from_file("alice") alice_pub = import_rsa_key_from_file("alice.pub") # Copy, sign and dump sub layout as link from template layout_template = Metablock.load("demo.layout.template") sub_layout = copy.deepcopy(layout_template) sub_layout_name = "sub_layout" sub_layout_path = FILENAME_FORMAT.format(step_name=sub_layout_name, keyid=alice_pub["keyid"]) sub_layout.sign(alice) sub_layout.dump(sub_layout_path) # Create super layout that has only one step, the sublayout self.super_layout = Layout() self.super_layout.keys[alice_pub["keyid"]] = alice_pub sub_layout_step = Step(name=sub_layout_name, pubkeys=[alice_pub["keyid"]]) self.super_layout.steps.append(sub_layout_step) # Load the super layout links (i.e. the sublayout) self.super_layout_links = load_links_for_layout(self.super_layout)
def _sign_and_dump_metadata(metadata, args): """ <Purpose> Internal method to sign link or layout metadata and dump it to disk. <Arguments> metadata: Metablock object (contains Link or Layout object) args: see argparser <Exceptions> SystemExit(0) if signing is successful SystemExit(2) if any exception occurs """ try: if not args.append: metadata.signatures = [] signature = None # If the cli tool was called with `--gpg [KEYID ...]` `args.gpg` is # a list (not None) and we will try to sign with gpg. # If `--gpg-home` was not set, args.gpg_home is None and the signer tries # to use the default gpg keyring. if args.gpg is not None: # If `--gpg` was passed without argument we sign with the default key # Excluded so that coverage does not vary in different test environments if len(args.gpg) == 0: # pragma: no cover signature = metadata.sign_gpg(gpg_keyid=None, gpg_home=args.gpg_home) # Otherwise we sign with each passed keyid for keyid in args.gpg: securesystemslib.formats.KEYID_SCHEMA.check_match(keyid) signature = metadata.sign_gpg(gpg_keyid=keyid, gpg_home=args.gpg_home) # Alternatively we iterate over passed private key paths `--key KEYPATH # ...` load the corresponding key from disk and sign with it elif args.key is not None: # pragma: no branch if args.key_type is None: args.key_type = [util.KEY_TYPE_RSA] * len(args.key) if len(args.key_type) != len(args.key): raise securesystemslib.exceptions.FormatError( "number of key_types should match with the number" " of keys specified") for idx, key_path in enumerate(args.key): key = util.import_private_key_from_file( key_path, args.key_type[idx]) signature = metadata.sign(key) # If `--output` was specified we store the signed link or layout metadata # to that location no matter what if args.output: out_path = args.output # Otherwise, in case of links, we build the filename using the link/step # name and the keyid of the created signature (there is only one for links) elif metadata.type_ == "link": securesystemslib.formats.ANY_SIGNATURE_SCHEMA.check_match( signature) keyid = signature["keyid"] out_path = FILENAME_FORMAT.format(step_name=metadata.signed.name, keyid=keyid) # In case of layouts we just override the input file. elif metadata.type_ == "layout": # pragma: no branch out_path = args.file LOG.info("Dumping {0} to '{1}'...".format(metadata.type_, out_path)) metadata.dump(out_path) sys.exit(0) except Exception as e: LOG.error("The following error occurred while signing: " "{}".format(e)) sys.exit(2)
def in_toto_run(name, material_list, product_list, link_cmd_args, key=False, record_streams=False): """ <Purpose> Calls function to run command passed as link_cmd_args argument, storing its materials, by-products and return value, and products into a link metadata file. The link metadata file is signed with the passed key and stored to disk. <Arguments> name: A unique name to relate link metadata with a step or inspection defined in the layout. material_list: List of file or directory paths that should be recorded as materials. product_list: List of file or directory paths that should be recorded as products. link_cmd_args: A list where the first element is a command and the remaining elements are arguments passed to that command. key: (optional) Private key to sign link metadata. Format is securesystemslib.formats.KEY_SCHEMA record_streams: (optional) A bool that specifies whether to redirect standard output and and standard error to a temporary file which is returned to the caller (True) or not (False). <Exceptions> None. <Side Effects> Writes newly created link metadata file to disk using the filename scheme from link.FILENAME_FORMAT <Returns> Newly created Metablock object containing a Link object """ log.info("Running '{}'...".format(name)) # If a key is passed, it has to match the format if key: securesystemslib.formats.KEY_SCHEMA.check_match(key) #FIXME: Add private key format check to securesystemslib formats if not key["keyval"].get("private"): raise securesystemslib.exceptions.FormatError( "Signing key needs to be a private key.") if material_list: log.info("Recording materials '{}'...".format( ", ".join(material_list))) materials_dict = record_artifacts_as_dict(material_list) if link_cmd_args: log.info("Running command '{}'...".format(" ".join(link_cmd_args))) byproducts = execute_link(link_cmd_args, record_streams) else: byproducts = {} if product_list: log.info("Recording products '{}'...".format(", ".join(product_list))) products_dict = record_artifacts_as_dict(product_list) log.info("Creating link metadata...") link = in_toto.models.link.Link(name=name, materials=materials_dict, products=products_dict, command=link_cmd_args, byproducts=byproducts, environment={"workdir": os.getcwd()}) link_metadata = Metablock(signed=link) if key: log.info("Signing link metadata with key '{:.8}...'...".format( key["keyid"])) link_metadata.sign(key) filename = FILENAME_FORMAT.format(step_name=name, keyid=key["keyid"]) log.info("Storing link metadata to '{}'...".format(filename)) link_metadata.dump(filename) return link_metadata
def in_toto_record_stop(step_name, product_list, signing_key=None, gpg_keyid=None, gpg_use_default=False, gpg_home=None, exclude_patterns=None, base_path=None, normalize_line_endings=False, lstrip_paths=None, metadata_directory=None): """Finalizes preliminary link metadata generated with in_toto_record_start. Loads preliminary link metadata file, verifies its signature, and records paths and hashes as products, thus finalizing the link metadata. The metadata is signed with the passed signing_key, a gpg key identified by its ID, or the default gpg key. If multiple key arguments are passed, only one key is used in above order of precedence. At least one key argument must be passed and it must be the same as the one used to sign the preliminary link metadata file. The resulting link file is written to ``STEP-NAME.KEYID-PREFIX.link``. Use this function together with in_toto_record_start as an alternative to in_toto_run, in order to provide evidence for supply chain steps that cannot be carried out by a single command. Arguments: step_name: A unique name to associate link metadata with a step. product_list: A list of artifact paths to be recorded as products. Directories are traversed recursively. signing_key (optional): A key used to sign the resulting link metadata. The format is securesystemslib.formats.KEY_SCHEMA. gpg_keyid (optional): A keyid used to identify a local gpg key used to sign the resulting link metadata. gpg_use_default (optional): A boolean indicating if the default gpg key should be used to sign the resulting link metadata. gpg_home (optional): A path to the gpg home directory. If not set the default gpg home directory is used. exclude_patterns (optional): A list of filename patterns to exclude certain files from being recorded as artifacts. base_path (optional): A path relative to which artifacts are recorded. Default is the current working directory. normalize_line_endings (optional): A boolean indicating if line endings of artifacts should be normalized before hashing for cross-platform support. lstrip_paths (optional): A list of path prefixes used to left-strip artifact paths before storing them in the resulting link metadata. metadata_directory (optional): A directory path to write the resulting link metadata file to. Default destination is the current working directory. Raises: securesystemslib.exceptions.FormatError: Passed arguments are malformed. ValueError: None of signing_key, gpg_keyid or gpg_use_default=True is passed. LinkNotFoundError: No preliminary link metadata file found. securesystemslib.exceptions.StorageError: Cannot hash artifacts. PrefixError: Left-stripping artifact paths results in non-unique dict keys. securesystemslib.process.subprocess.TimeoutExpired: Link command times out. IOError, FileNotFoundError, NotADirectoryError, PermissionError: Cannot write link metadata. securesystemslib.exceptions.CryptoError, \ securesystemslib.exceptions.UnsupportedAlgorithmError: Signing errors. ValueError, OSError, securesystemslib.gpg.exceptions.CommandError, \ securesystemslib.gpg.exceptions.KeyNotFoundError: gpg signing errors. Side Effects: Reads preliminary link metadata file from disk. Reads artifact files from disk. Calls system gpg in a subprocess, if a gpg key argument is passed. Writes resulting link metadata file to disk. Removes preliminary link metadata file from disk. """ LOG.info("Stop recording '{}'...".format(step_name)) # Check that we have something to sign and if the formats are right if not signing_key and not gpg_keyid and not gpg_use_default: raise ValueError("Pass either a signing key, a gpg keyid or set" " gpg_use_default to True") if signing_key: _check_match_signing_key(signing_key) if gpg_keyid: securesystemslib.formats.KEYID_SCHEMA.check_match(gpg_keyid) if exclude_patterns: securesystemslib.formats.NAMES_SCHEMA.check_match(exclude_patterns) if base_path: securesystemslib.formats.PATH_SCHEMA.check_match(base_path) if metadata_directory: securesystemslib.formats.PATH_SCHEMA.check_match(metadata_directory) # Load preliminary link file # If we have a signing key we can use the keyid to construct the name if signing_key: unfinished_fn = UNFINISHED_FILENAME_FORMAT.format( step_name=step_name, keyid=signing_key["keyid"]) # FIXME: Currently there is no way to know the default GPG key's keyid and # so we glob for preliminary link files else: unfinished_fn_glob = UNFINISHED_FILENAME_FORMAT_GLOB.format( step_name=step_name, pattern="*") unfinished_fn_list = glob.glob(unfinished_fn_glob) if not len(unfinished_fn_list): raise in_toto.exceptions.LinkNotFoundError( "Could not find a preliminary" " link for step '{}' in the current working directory.".format( step_name)) if len(unfinished_fn_list) > 1: raise in_toto.exceptions.LinkNotFoundError( "Found more than one" " preliminary links for step '{}' in the current working directory:" " {}. We need exactly one to stop recording.".format( step_name, ", ".join(unfinished_fn_list))) unfinished_fn = unfinished_fn_list[0] LOG.info("Loading preliminary link metadata '{}'...".format(unfinished_fn)) link_metadata = Metablock.load(unfinished_fn) # The file must have been signed by the same key # If we have a signing_key we use it for verification as well if signing_key: LOG.info( "Verifying preliminary link signature using passed signing key...") keyid = signing_key["keyid"] verification_key = signing_key elif gpg_keyid: LOG.info( "Verifying preliminary link signature using passed gpg key...") gpg_pubkey = securesystemslib.gpg.functions.export_pubkey( gpg_keyid, gpg_home) keyid = gpg_pubkey["keyid"] verification_key = gpg_pubkey else: # must be gpg_use_default # FIXME: Currently there is no way to know the default GPG key's keyid # before signing. As a workaround we extract the keyid of the preliminary # Link file's signature and try to export a pubkey from the gpg # home directory. We do this even if a gpg_keyid was specified, because gpg # accepts many different ids (mail, name, parts of an id, ...) but we # need a specific format. LOG.info( "Verifying preliminary link signature using default gpg key...") keyid = link_metadata.signatures[0]["keyid"] gpg_pubkey = securesystemslib.gpg.functions.export_pubkey( keyid, gpg_home) verification_key = gpg_pubkey link_metadata.verify_signature(verification_key) # Record products if a product path list was passed if product_list: LOG.info("Recording products '{}'...".format(", ".join(product_list))) link_metadata.signed.products = record_artifacts_as_dict( product_list, exclude_patterns=exclude_patterns, base_path=base_path, follow_symlink_dirs=True, normalize_line_endings=normalize_line_endings, lstrip_paths=lstrip_paths) link_metadata.signatures = [] if signing_key: LOG.info("Updating signature with key '{:.8}...'...".format(keyid)) link_metadata.sign(signing_key) else: # gpg_keyid or gpg_use_default # In both cases we use the keyid we got from verifying the preliminary # link signature above. LOG.info("Updating signature with gpg key '{:.8}...'...".format(keyid)) link_metadata.sign_gpg(keyid, gpg_home) fn = FILENAME_FORMAT.format(step_name=step_name, keyid=keyid) if metadata_directory is not None: fn = os.path.join(metadata_directory, fn) LOG.info("Storing link metadata to '{}'...".format(fn)) link_metadata.dump(fn) LOG.info("Removing unfinished link metadata '{}'...".format(unfinished_fn)) os.remove(unfinished_fn)
def in_toto_run(name, material_list, product_list, link_cmd_args, record_streams=False, signing_key=None, gpg_keyid=None, gpg_use_default=False, gpg_home=None, exclude_patterns=None, base_path=None, compact_json=False, record_environment=False, normalize_line_endings=False, lstrip_paths=None, metadata_directory=None): """Performs a supply chain step or inspection generating link metadata. Executes link_cmd_args, recording paths and hashes of files before and after command execution (aka. artifacts) in a link metadata file. The metadata is signed with the passed signing_key, a gpg key identified by its ID, or the default gpg key. If multiple key arguments are passed, only one key is used in above order of precedence. The resulting link file is written to ``STEP-NAME.KEYID-PREFIX.link``. If no key argument is passed the link metadata is neither signed nor written to disk. Arguments: name: A unique name to associate link metadata with a step or inspection. material_list: A list of artifact paths to be recorded before command execution. Directories are traversed recursively. product_list: A list of artifact paths to be recorded after command execution. Directories are traversed recursively. link_cmd_args: A list where the first element is a command and the remaining elements are arguments passed to that command. record_streams (optional): A boolean indicating if standard output and standard error of the link command should be recorded in the link metadata in addition to being displayed while the command is executed. signing_key (optional): A key used to sign the resulting link metadata. The format is securesystemslib.formats.KEY_SCHEMA. gpg_keyid (optional): A keyid used to identify a local gpg key used to sign the resulting link metadata. gpg_use_default (optional): A boolean indicating if the default gpg key should be used to sign the resulting link metadata. gpg_home (optional): A path to the gpg home directory. If not set the default gpg home directory is used. exclude_patterns (optional): A list of filename patterns to exclude certain files from being recorded as artifacts. See Config docs for details. base_path (optional): A path relative to which artifacts are recorded. Default is the current working directory. compact_json (optional): A boolean indicating if the resulting link metadata should be written in the most compact JSON representation. record_environment (optional): A boolean indicating if information about the environment should be added in the resulting link metadata. normalize_line_endings (optional): A boolean indicating if line endings of artifacts should be normalized before hashing for cross-platform support. lstrip_paths (optional): A list of path prefixes used to left-strip artifact paths before storing them in the resulting link metadata. metadata_directory (optional): A directory path to write the resulting link metadata file to. Default destination is the current working directory. Raises: securesystemslib.exceptions.FormatError: Passed arguments are malformed. ValueError: Cannot change to base path directory. securesystemslib.exceptions.StorageError: Cannot hash artifacts. PrefixError: Left-stripping artifact paths results in non-unique dict keys. securesystemslib.process.subprocess.TimeoutExpired: Link command times out. IOError, FileNotFoundError, NotADirectoryError, PermissionError: Cannot write link metadata. securesystemslib.exceptions.CryptoError, \ securesystemslib.exceptions.UnsupportedAlgorithmError: Signing errors. ValueError, OSError, securesystemslib.gpg.exceptions.CommandError, \ securesystemslib.gpg.exceptions.KeyNotFoundError: gpg signing errors. Side Effects: Reads artifact files from disk. Runs link command in subprocess. Calls system gpg in a subprocess, if a gpg key argument is passed. Writes link metadata file to disk, if any key argument is passed. Returns: A Metablock object that contains the resulting link object. """ LOG.info("Running '{}'...".format(name)) # Check key formats to fail early if signing_key: _check_match_signing_key(signing_key) if gpg_keyid: securesystemslib.formats.KEYID_SCHEMA.check_match(gpg_keyid) if exclude_patterns: securesystemslib.formats.NAMES_SCHEMA.check_match(exclude_patterns) if base_path: securesystemslib.formats.PATH_SCHEMA.check_match(base_path) if metadata_directory: securesystemslib.formats.PATH_SCHEMA.check_match(metadata_directory) if material_list: LOG.info("Recording materials '{}'...".format( ", ".join(material_list))) materials_dict = record_artifacts_as_dict( material_list, exclude_patterns=exclude_patterns, base_path=base_path, follow_symlink_dirs=True, normalize_line_endings=normalize_line_endings, lstrip_paths=lstrip_paths) if link_cmd_args: LOG.info("Running command '{}'...".format(" ".join(link_cmd_args))) byproducts = execute_link(link_cmd_args, record_streams) else: byproducts = {} if product_list: securesystemslib.formats.PATHS_SCHEMA.check_match(product_list) LOG.info("Recording products '{}'...".format(", ".join(product_list))) products_dict = record_artifacts_as_dict( product_list, exclude_patterns=exclude_patterns, base_path=base_path, follow_symlink_dirs=True, normalize_line_endings=normalize_line_endings, lstrip_paths=lstrip_paths) LOG.info("Creating link metadata...") environment = {} if record_environment: environment['workdir'] = os.getcwd().replace('\\', '/') link = in_toto.models.link.Link(name=name, materials=materials_dict, products=products_dict, command=link_cmd_args, byproducts=byproducts, environment=environment) link_metadata = Metablock(signed=link, compact_json=compact_json) signature = None if signing_key: LOG.info("Signing link metadata using passed key...") signature = link_metadata.sign(signing_key) elif gpg_keyid: LOG.info("Signing link metadata using passed GPG keyid...") signature = link_metadata.sign_gpg(gpg_keyid, gpg_home=gpg_home) elif gpg_use_default: LOG.info("Signing link metadata using default GPG key ...") signature = link_metadata.sign_gpg(gpg_keyid=None, gpg_home=gpg_home) # We need the signature's keyid to write the link to keyid infix'ed filename if signature: signing_keyid = signature["keyid"] filename = FILENAME_FORMAT.format(step_name=name, keyid=signing_keyid) if metadata_directory is not None: filename = os.path.join(metadata_directory, filename) LOG.info("Storing link metadata to '{}'...".format(filename)) link_metadata.dump(filename) return link_metadata
def in_toto_record_stop(step_name, product_list, signing_key=None, gpg_keyid=None, gpg_use_default=False, gpg_home=None, exclude_patterns=None, base_path=None, normalize_line_endings=False): """ <Purpose> Finishes creating link metadata for a multi-part in-toto step. Loads unfinished link metadata file from disk, verifies that the file was signed with either the passed signing key, a gpg key identified by the passed gpg_keyid or the default gpg key. Then records products, updates unfinished Link object (products and signature), removes unfinished link file from and stores new link file to disk. One of signing_key, gpg_keyid or gpg_use_default has to be passed and it needs to be the same that was used with preceding in_toto_record_start. <Arguments> step_name: A unique name to relate link metadata with a step defined in the layout. product_list: List of file or directory paths that should be recorded as products. signing_key: (optional) If not None, link metadata is signed with this key. Format is securesystemslib.formats.KEY_SCHEMA gpg_keyid: (optional) If not None, link metadata is signed with a gpg key identified by the passed keyid. gpg_use_default: (optional) If True, link metadata is signed with default gpg key. gpg_home: (optional) Path to GPG keyring (if not set the default keyring is used). exclude_patterns: (optional) Artifacts matched by the pattern are excluded from the products sections in the resulting link. base_path: (optional) If passed, record products relative to base_path. Default is current working directory. NOTE: The base_path part of the recorded products is not included in the resulting preliminary link's product section. normalize_line_endings: (optional) If True, replaces windows and mac line endings with unix line endings before hashing products, for cross-platform support. <Exceptions> ValueError if none of signing_key, gpg_keyid or gpg_use_default=True is passed. securesystemslib.FormatError if a signing_key is passed and does not match securesystemslib.formats.KEY_SCHEMA or a gpg_keyid is passed and does not match securesystemslib.formats.KEYID_SCHEMA, or exclude_patterns are passed and don't match securesystemslib.formats.NAMES_SCHEMA, or base_path is passed and does not match securesystemslib.formats.PATH_SCHEMA or is not a directory. LinkNotFoundError if gpg is used for signing and the corresponding preliminary link file can not be found in the current working directory <Side Effects> Writes newly created link metadata file to disk using the filename scheme from link.FILENAME_FORMAT Removes unfinished link file link.UNFINISHED_FILENAME_FORMAT from disk <Returns> None. """ log.info("Stop recording '{}'...".format(step_name)) # Check that we have something to sign and if the formats are right if not signing_key and not gpg_keyid and not gpg_use_default: raise ValueError("Pass either a signing key, a gpg keyid or set" " gpg_use_default to True") if signing_key: _check_match_signing_key(signing_key) if gpg_keyid: securesystemslib.formats.KEYID_SCHEMA.check_match(gpg_keyid) if exclude_patterns: securesystemslib.formats.NAMES_SCHEMA.check_match(exclude_patterns) if base_path: securesystemslib.formats.PATH_SCHEMA.check_match(base_path) # Load preliminary link file # If we have a signing key we can use the keyid to construct the name if signing_key: unfinished_fn = UNFINISHED_FILENAME_FORMAT.format( step_name=step_name, keyid=signing_key["keyid"]) # FIXME: Currently there is no way to know the default GPG key's keyid and # so we glob for preliminary link files else: unfinished_fn_glob = UNFINISHED_FILENAME_FORMAT_GLOB.format( step_name=step_name, pattern="*") unfinished_fn_list = glob.glob(unfinished_fn_glob) if not len(unfinished_fn_list): raise in_toto.exceptions.LinkNotFoundError( "Could not find a preliminary" " link for step '{}' in the current working directory.".format( step_name)) if len(unfinished_fn_list) > 1: raise in_toto.exceptions.LinkNotFoundError( "Found more than one" " preliminary links for step '{}' in the current working directory:" " {}. We need exactly one to stop recording.".format( step_name, ", ".join(unfinished_fn_list))) unfinished_fn = unfinished_fn_list[0] log.info("Loading preliminary link metadata '{}'...".format(unfinished_fn)) link_metadata = Metablock.load(unfinished_fn) # The file must have been signed by the same key # If we have a signing_key we use it for verification as well if signing_key: log.info( "Verifying preliminary link signature using passed signing key...") keyid = signing_key["keyid"] verification_key = signing_key elif gpg_keyid: log.info( "Verifying preliminary link signature using passed gpg key...") gpg_pubkey = in_toto.gpg.functions.gpg_export_pubkey( gpg_keyid, gpg_home) keyid = gpg_pubkey["keyid"] verification_key = gpg_pubkey else: # must be gpg_use_default # FIXME: Currently there is no way to know the default GPG key's keyid # before signing. As a workaround we extract the keyid of the preliminary # Link file's signature and try to export a pubkey from the gpg # keyring. We do this even if a gpg_keyid was specified, because gpg # accepts many different ids (mail, name, parts of an id, ...) but we # need a specific format. log.info( "Verifying preliminary link signature using default gpg key...") keyid = link_metadata.signatures[0]["keyid"] gpg_pubkey = in_toto.gpg.functions.gpg_export_pubkey(keyid, gpg_home) verification_key = gpg_pubkey link_metadata.verify_signature(verification_key) # Record products if a product path list was passed if product_list: log.info("Recording products '{}'...".format(", ".join(product_list))) link_metadata.signed.products = record_artifacts_as_dict( product_list, exclude_patterns=exclude_patterns, base_path=base_path, follow_symlink_dirs=True, normalize_line_endings=normalize_line_endings) link_metadata.signatures = [] if signing_key: log.info("Updating signature with key '{:.8}...'...".format(keyid)) link_metadata.sign(signing_key) else: # gpg_keyid or gpg_use_default # In both cases we use the keyid we got from verifying the preliminary # link signature above. log.info("Updating signature with gpg key '{:.8}...'...".format(keyid)) link_metadata.sign_gpg(keyid, gpg_home) fn = FILENAME_FORMAT.format(step_name=step_name, keyid=keyid) log.info("Storing link metadata to '{}'...".format(fn)) link_metadata.dump(fn) log.info("Removing unfinished link metadata '{}'...".format(unfinished_fn)) os.remove(unfinished_fn)
def in_toto_run(name, material_list, product_list, link_cmd_args, record_streams=False, signing_key=None, gpg_keyid=None, gpg_use_default=False, gpg_home=None, exclude_patterns=None, base_path=None, compact_json=False, record_environment=False, normalize_line_endings=False): """ <Purpose> Calls functions in this module to run the command passed as link_cmd_args argument and to store materials, products, by-products and environment information into a link metadata file. The link metadata file is signed either with the passed signing_key, or a gpg key identified by the passed gpg_keyid or with the default gpg key if gpg_use_default is True. Even if multiple key parameters are passed, only one key is used for signing (in above order of precedence). The link file is dumped to `link.FILENAME_FORMAT` using the signing key's keyid. If no key parameter is passed the link is neither signed nor dumped. <Arguments> name: A unique name to relate link metadata with a step or inspection defined in the layout. material_list: List of file or directory paths that should be recorded as materials. product_list: List of file or directory paths that should be recorded as products. link_cmd_args: A list where the first element is a command and the remaining elements are arguments passed to that command. record_streams: (optional) A bool that specifies whether to redirect standard output and and standard error to a temporary file which is returned to the caller (True) or not (False). signing_key: (optional) If not None, link metadata is signed with this key. Format is securesystemslib.formats.KEY_SCHEMA gpg_keyid: (optional) If not None, link metadata is signed with a gpg key identified by the passed keyid. gpg_use_default: (optional) If True, link metadata is signed with default gpg key. gpg_home: (optional) Path to GPG keyring (if not set the default keyring is used). exclude_patterns: (optional) Artifacts matched by the pattern are excluded from the materials and products sections in the resulting link. base_path: (optional) If passed, record artifacts relative to base_path. Default is current working directory. NOTE: The base_path part of the recorded material is not included in the resulting preliminary link's material/product sections. compact_json: (optional) Whether or not to use the most compact json representation. record_environment: (optional) if values such as workdir should be recorded on the environment dictionary (false by default) normalize_line_endings: (optional) If True, replaces windows and mac line endings with unix line endings before hashing materials and products, for cross-platform support. <Exceptions> securesystemslib.FormatError if a signing_key is passed and does not match securesystemslib.formats.KEY_SCHEMA or a gpg_keyid is passed and does not match securesystemslib.formats.KEYID_SCHEMA or exclude_patterns are passed and don't match securesystemslib.formats.NAMES_SCHEMA, or base_path is passed and does not match securesystemslib.formats.PATH_SCHEMA or is not a directory. <Side Effects> If a key parameter is passed for signing, the newly created link metadata file is written to disk using the filename scheme: `link.FILENAME_FORMAT` <Returns> Newly created Metablock object containing a Link object """ log.info("Running '{}'...".format(name)) # Check key formats to fail early if signing_key: _check_match_signing_key(signing_key) if gpg_keyid: securesystemslib.formats.KEYID_SCHEMA.check_match(gpg_keyid) if exclude_patterns: securesystemslib.formats.NAMES_SCHEMA.check_match(exclude_patterns) if base_path: securesystemslib.formats.PATH_SCHEMA.check_match(base_path) if material_list: log.info("Recording materials '{}'...".format( ", ".join(material_list))) materials_dict = record_artifacts_as_dict( material_list, exclude_patterns=exclude_patterns, base_path=base_path, follow_symlink_dirs=True, normalize_line_endings=normalize_line_endings) if link_cmd_args: log.info("Running command '{}'...".format(" ".join(link_cmd_args))) byproducts = execute_link(link_cmd_args, record_streams) else: byproducts = {} if product_list: securesystemslib.formats.PATHS_SCHEMA.check_match(product_list) log.info("Recording products '{}'...".format(", ".join(product_list))) products_dict = record_artifacts_as_dict( product_list, exclude_patterns=exclude_patterns, base_path=base_path, follow_symlink_dirs=True, normalize_line_endings=normalize_line_endings) log.info("Creating link metadata...") environment = {} if record_environment: environment['workdir'] = os.getcwd().replace('\\', '/') link = in_toto.models.link.Link(name=name, materials=materials_dict, products=products_dict, command=link_cmd_args, byproducts=byproducts, environment=environment) link_metadata = Metablock(signed=link, compact_json=compact_json) signature = None if signing_key: log.info("Signing link metadata using passed key...") signature = link_metadata.sign(signing_key) elif gpg_keyid: log.info("Signing link metadata using passed GPG keyid...") signature = link_metadata.sign_gpg(gpg_keyid, gpg_home=gpg_home) elif gpg_use_default: log.info("Signing link metadata using default GPG key ...") signature = link_metadata.sign_gpg(gpg_keyid=None, gpg_home=gpg_home) # We need the signature's keyid to write the link to keyid infix'ed filename if signature: signing_keyid = signature["keyid"] filename = FILENAME_FORMAT.format(step_name=name, keyid=signing_keyid) log.info("Storing link metadata to '{}'...".format(filename)) link_metadata.dump(filename) return link_metadata