def main():
"""Parse arguments, load key from disk (prompts for password if key is
encrypted) and call in_toto_run. """
parser = create_parser()
args = parser.parse_args()
LOG.setLevelVerboseOrQuiet(args.verbose, args.quiet)
# Override defaults in settings.py with environment variables and RCfiles
in_toto.user_settings.set_settings()
# Regular signing and GPG signing are mutually exclusive
if (args.key is None) == (args.gpg is None):
parser.print_usage()
parser.error("Specify either '--key <key path>' or '--gpg [<keyid>]'")
# If `--gpg` was set without argument it has the value `True` and
# we will try to sign with the default key
gpg_use_default = (args.gpg is True)
# Otherwise we interpret it as actual keyid
gpg_keyid = None
if args.gpg is not True:
gpg_keyid = args.gpg
# If no_command is specified run in_toto_run without executing a command
if args.no_command:
args.link_cmd = []
elif not args.link_cmd: # pragma: no branch
parser.print_usage()
parser.error("No command specified."
" Please specify (or use the --no-command option)")
try:
# We load the key here because it might prompt the user for a password in
# case the key is encrypted. Something that should not happen in the lib.
key = None
if args.key:
key = util.import_private_key_from_file(args.key, args.key_type)
runlib.in_toto_run(args.step_name,
args.materials,
args.products,
args.link_cmd,
record_streams=args.record_streams,
signing_key=key,
gpg_keyid=gpg_keyid,
gpg_use_default=gpg_use_default,
gpg_home=args.gpg_home,
exclude_patterns=args.exclude_patterns,
base_path=args.base_path,
lstrip_paths=args.lstrip_paths,
metadata_directory=args.metadata_directory)
except Exception as e:
LOG.error("(in-toto-run) {0}: {1}".format(type(e).__name__, e))
sys.exit(1)
sys.exit(0)
def test_in_toto_read_only_metadata_directory(self):
"""Fail run, passed metadata directory is read only"""
tmp_dir = os.path.realpath(tempfile.mkdtemp())
# make the directory read only
os.chmod(tmp_dir, stat.S_IREAD)
with self.assertRaises(PermissionError):
in_toto_run(self.step_name, None, None, ["python", "--version"],
True, self.key, metadata_directory=tmp_dir)
os.rmdir(tmp_dir)
def test_nonexistent_directory(self):
"""Fail run, passed metadata_directory not exist. """
with self.assertRaises(FileNotFoundError):
in_toto_run(self.step_name,
None,
None, ["python", "--version"],
True,
self.key,
metadata_directory='nonexistentDir')
def test_not_a_directory(self):
"""Fail run, passed metadata_directory is not a directory. """
fd, path = tempfile.mkstemp()
os.write(fd, b"hello in-toto")
os.close(fd)
# Windows will raise FileNotFoundError instead of NotADirectoryError
with self.assertRaises((NotADirectoryError, FileNotFoundError)):
in_toto_run(self.step_name, None, None, ["python", "--version"],
True, self.key, metadata_directory=path)
os.remove(path)
def test_nonexistent_directory(self):
"""Fail run, passed metadata_directory not exist. """
expected_error = IOError if sys.version_info < (3, 0) \
else FileNotFoundError
with self.assertRaises(expected_error):
in_toto_run(self.step_name,
None,
None, ["python", "--version"],
True,
self.key,
metadata_directory='nonexistentDir')
def test_in_toto_run_compare_with_and_without_metadata_directory(self):
"""Successfully run with and without metadata directory,
compare the signed is equal"""
tmp_dir = os.path.realpath(tempfile.mkdtemp(dir=os.getcwd()))
in_toto_run(self.step_name, [self.test_artifact], [self.test_artifact],
["python", "--version"], True, self.key, metadata_directory=tmp_dir)
file_path = os.path.join(tmp_dir, FILENAME_FORMAT.format(step_name=self.step_name,
keyid=self.key["keyid"]))
link_dump_with_md = Metablock.load(file_path)
in_toto_run(self.step_name, [self.test_artifact], [self.test_artifact],
["python", "--version"], True, self.key)
link_dump_without_md = Metablock.load(
FILENAME_FORMAT.format(step_name=self.step_name, keyid=self.key["keyid"]))
self.assertEqual(repr(link_dump_with_md.signed),
repr(link_dump_without_md.signed))
def test_in_toto_run_with_byproduct(self):
"""Successfully run, verify recorded byproduct. """
link = in_toto_run(self.step_name,
None,
None, ["echo", "test"],
record_streams=True)
self.assertTrue("test" in link.signed.byproducts.get("stdout"))
def test_normalize_line_endings(self):
"""Test cross-platform line ending normalization. """
paths = []
try:
# Create three artifacts with same content but different line endings
for line_ending in [b"\n", b"\r", b"\r\n"]:
fd, path = tempfile.mkstemp()
paths.append(path)
os.write(fd, b"hello" + line_ending + b"toto")
os.close(fd)
# Call in_toto_run and record artifacts as materials and products
# with line ending normalization on
link = in_toto_run(self.step_name,
paths,
paths, ["python", "--version"],
normalize_line_endings=True).signed
# Check that all three hashes in materials and products are equal
for artifact_dict in [link.materials, link.products]:
hash_dicts = list(artifact_dict.values())
self.assertTrue(hash_dicts[1:] == hash_dicts[:-1])
# Clean up
finally:
for path in paths:
os.remove(path)
def test_in_toto_run_compare_dumped_with_returned_link(self):
"""Successfully run, compare dumped link is equal to returned link. """
link = in_toto_run(self.step_name, [self.test_artifact],
[self.test_artifact], ["python", "--version"], True, self.key)
link_dump = Metablock.load(
FILENAME_FORMAT.format(step_name=self.step_name, keyid=self.key["keyid"]))
self.assertEqual(repr(link), repr(link_dump))
def test_in_toto_run_without_byproduct(self):
"""Successfully run, verify byproduct is not recorded. """
link = in_toto_run(self.step_name,
None,
None, ["python", "--version"],
record_streams=False)
self.assertFalse(len(link.signed.byproducts.get("stdout")))
def test_in_toto_run_verify_recorded_artifacts(self):
"""Successfully run, verify properly recorded artifacts. """
link = in_toto_run(self.step_name, [self.test_artifact],
[self.test_artifact], ["python", "--version"])
self.assertEqual(list(link.signed.materials.keys()),
list(link.signed.products.keys()),
[self.test_artifact])
def runInToto(keyId):
def load_gitignore(filename = '.gitignore'):
exclude_patterns = []
with open(filename) as gitignore:
for line in gitignore:
line = line.strip()
if line and not line.startswith('#'):
exclude_patterns.append(line)
return exclude_patterns
# NOTE: Exclude anything that looks like the following patterns.
exclude_patterns = list(set(in_toto.settings.ARTIFACT_EXCLUDE_PATTERNS + \
load_gitignore()))
# TODO: Would be nice to pass to in-toto the GPG executable to call.
inTotoCmd = runlib.in_toto_run(
# Do NOT record files matching these patters.
exclude_patterns = exclude_patterns,
# Use this GPG key.
gpg_keyid = keyId,
# Do NOT execute any other command.
link_cmd_args = [],
# Do NOT record anything as input.
material_list = None,
# Use this step name.
name = STEP_NAME,
# Do record every source file, except for exclude_patterns, as output.
product_list = "."
)
def main():
args = parse_arguments()
# configure our instance of the grafeas api
swagger_client.configuration.host = args.target
api_instance = swagger_client.GrafeasApi()
project_id = args.project_id
try:
# Create and return an unsigned link (not dumping to disk)
link = runlib.in_toto_run(args.name, args.materials, args.products,
args.link_cmd)
# Now sign the link with the passed key
key = util.import_rsa_key_from_file(args.key)
link.sign(key)
except Exception as e:
print("Exception when calling in-toto runlib: {}".format(e))
sys.exit(1)
# Create an occurrence from the link
link_dict = attr.asdict(link)
# We need to cast return-value to string or else parsing breaks mysteriously
link_dict["signed"]["byproducts"]["return-value"] = \
str(link_dict["signed"]["byproducts"]["return-value"])
# Create the in-toto link extended occurrence metadata
signable = swagger_client.LinkSignable(**(link_dict["signed"]))
grafeas_link = swagger_client.LinkMetadata(signable, link.signatures)
# Create the occurrence object that is posted to the server
occurrence = swagger_client.Occurrence()
# Occurrences/links use the step name + signing keyid as id
# There can be multiple occurrences per note (links per step)
occurrence_id = "{}-{}".format(args.name, link.signatures[0]["keyid"][:8])
occurrence_name = "projects/{}/occurrences/{}".format(
project_id, occurrence_id)
# Notes/steps only use the step name as id
note_name = "projects/{}/notes/{}".format(project_id, args.name)
occurrence.name = occurrence_name
occurrence.note_name = note_name
occurrence.link_metadata = grafeas_link
try:
api_response = api_instance.create_occurrence(project_id,
occurrence=occurrence)
pprint(api_response)
except ApiException as e:
print(
"Exception when calling GrafeasApi->create_occurrence: {}".format(
e))
sys.exit(1)
def in_toto_run(step_name, material_list, product_list, link_cmd_args, key,
record_streams):
"""
<Purpose>
Calls runlib.in_toto_run and catches exceptions
<Arguments>
step_name:
A unique name to relate link metadata with a step defined in the
layout.
material_list:
List of file or directory paths that should be recorded as
materials.
product_list:
List of file or directory paths that should be recorded as
products.
link_cmd_args:
A list where the first element is a command and the remaining
elements are arguments passed to that command.
key:
Private key to sign link metadata.
Format is securesystemslib.formats.KEY_SCHEMA
record_streams:
A bool that specifies whether to redirect standard output and
and standard error to a temporary file which is returned to the
caller (True) or not (False).
<Exceptions>
SystemExit if any exception occurs
<Side Effects>
Calls sys.exit(1) if an exception is raised
<Returns>
None.
"""
"""Load link signing private keys from disk and runs passed command, storing
its materials, by-products and return value, and products into link metadata
file. The link metadata file is signed and stored to disk. """
try:
runlib.in_toto_run(step_name, material_list, product_list,
link_cmd_args, key, record_streams)
except Exception as e:
log.error("in toto run - {}".format(e))
sys.exit(1)
def run_in_toto(products, **kwargs):
exclude_patterns = read_gitignore_patterns()
runlib.in_toto_run(
# Do not record files matching these patterns.
exclude_patterns=exclude_patterns,
# Do not execute any other command.
link_cmd_args=[],
# Do not record anything as input.
material_list=None,
# Use this step name.
name=STEP_NAME,
# Record every source file, except for exclude_patterns, as output.
product_list=products,
# Keep file size down
compact_json=True,
# Extra options
**kwargs,
)
def test_in_toto_run_with_metadata_directory(self):
"""Successfully run with metadata directory,
compare dumped link is equal to returned link"""
tmp_dir = os.path.realpath(tempfile.mkdtemp(dir=os.getcwd()))
link = in_toto_run(self.step_name, [self.test_artifact],
[self.test_artifact], ["python", "--version"], True, self.key,
metadata_directory=tmp_dir)
file_path = os.path.join(tmp_dir, FILENAME_FORMAT.format(step_name=self.step_name,
keyid=self.key["keyid"]))
link_dump = Metablock.load(file_path)
self.assertEqual(repr(link), repr(link_dump))
def run_in_toto(key_id, products):
exclude_patterns = read_gitignore_patterns()
runlib.in_toto_run(
# Do not record files matching these patterns.
exclude_patterns=exclude_patterns,
# Use this GPG key.
gpg_keyid=key_id,
# Do not execute any other command.
link_cmd_args=[],
# Do not record anything as input.
material_list=None,
# Use this step name.
name=STEP_NAME,
# Record every source file, except for exclude_patterns, as output.
product_list=products,
# Keep file size down
compact_json=True,
# Cross-platform support
normalize_line_endings=True,
)
def test_in_toto_run_with_byproduct(self):
"""Successfully run, verify recorded byproduct. """
link = in_toto_run(self.step_name,
None,
None, ["python", "--version"],
record_streams=True)
# this or clause may seem weird, but given that python 2 prints its version
# to stderr while python3 prints it to stdout we check on both (or add a
# more verbose if clause)
self.assertTrue("Python" in link.signed.byproducts.get("stdout")
or "Python" in link.signed.byproducts.get("stderr"))
def test_in_toto_wrong_key(self):
"""Fail run, passed key is a public key. """
with self.assertRaises(securesystemslib.exceptions.FormatError):
in_toto_run(self.step_name, None, None, ["python", "--version"],
True, self.key_pub)
def test_in_toto_bad_signing_key_format(self):
"""Fail run, passed key is not properly formatted. """
with self.assertRaises(securesystemslib.exceptions.FormatError):
in_toto_run(self.step_name, None, None, ["python", "--version"],
True, "this-is-not-a-key")
def test_in_toto_run_verify_workdir(self):
"""Successfully run, verify cwd. """
link = in_toto_run(self.step_name, [], [], ["python", "--version"])
self.assertEquals(link.signed.environment["workdir"], os.getcwd())
def main():
"""Parse arguments, load key from disk (prompts for password if key is
encrypted) and call in_toto_run. """
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""
Executes the passed command and records paths and hashes of 'materials' (i.e.
files before command execution) and 'products' (i.e. files after command
execution) and stores them together with other information (executed command,
return value, stdout, stderr, ...) to a link metadata file, which is signed
with the passed key. Returns nonzero value on failure and zero otherwise.""")
parser.usage = ("%(prog)s <named arguments> [optional arguments]"
" -- <command> [args]")
parser.epilog = """
examples:
Tag a git repo, storing files in CWD as products, signing the resulting link
file with the private key loaded from 'key_file'.
{prog} -n tag -p . -k key_file -- git tag v1.0
Create tarball, storing files in 'project' directory as materials and the
tarball as product, signing the link file with GPG key '...7E0C8A17'.
{prog} -n package -m project -p project.tar.gz \\
-g 8465A1E2E0FB2B40ADB2478E18FB3F537E0C8A17 \\
-- tar czf project.tar.gz project
""".format(prog=parser.prog)
named_args = parser.add_argument_group("required named arguments")
# FIXME: Do we limit the allowed characters for the name?
named_args.add_argument(
"-n",
"--step-name",
type=str,
required=True,
metavar="<name>",
help=("Name used to associate the resulting link metadata with the"
" corresponding step defined in an in-toto layout."))
parser.add_argument(
"-m",
"--materials",
type=str,
required=False,
nargs='+',
metavar="<path>",
help=
("Paths to files or directories, whose paths and hashes are stored in the"
" resulting link metadata before the command is executed. Symlinks are"
" followed."))
parser.add_argument(
"-p",
"--products",
type=str,
required=False,
nargs='+',
metavar="<path>",
help=
("Paths to files or directories, whose paths and hashes are stored in the"
" resulting link metadata after the command is executed. Symlinks are"
" followed."))
named_args.add_argument(
"-k",
"--key",
type=str,
metavar="<path>",
help=
("Path to a PEM formatted private key file used to sign the resulting"
" link metadata."
" (passing one of '--key' or '--gpg' is required)"))
parser.add_argument(
"-t",
"--key-type",
dest="key_type",
type=str,
choices=util.SUPPORTED_KEY_TYPES,
default=util.KEY_TYPE_RSA,
help=
("Specify the key-type of the key specified by the '--key' option. If"
" '--key-type' is not passed, default is \"ed25519\"."))
named_args.add_argument(
"-g",
"--gpg",
nargs="?",
const=True,
metavar="<id>",
help=
("GPG keyid used to sign the resulting link metadata. When '--gpg' is"
" passed without keyid, the keyring's default GPG key is used."
" (passing one of '--key' or '--gpg' is required)"))
parser.add_argument(
"--gpg-home",
dest="gpg_home",
type=str,
metavar="<path>",
help=
("Path to GPG keyring to load GPG key identified by '--gpg' option. If"
" '--gpg-home' is not passed, the default GPG keyring is used."))
parser.add_argument(
"-s",
"--record-streams",
dest="record_streams",
default=False,
action="store_true",
help=
("If passed 'stdout' and 'stderr' of the executed command are duplicated"
" and stored in the resulting link metadata."))
parser.add_argument(
"-x",
"--no-command",
dest="no_command",
default=False,
action="store_true",
help=
("Generate link metadata without executing a command, e.g. for a 'signed"
" off by' step."))
parser.add_argument(*EXCLUDE_ARGS, **EXCLUDE_KWARGS)
parser.add_argument(*BASE_PATH_ARGS, **BASE_PATH_KWARGS)
parser.add_argument(*LSTRIP_PATHS_ARGS, **LSTRIP_PATHS_KWARGS)
verbosity_args = parser.add_mutually_exclusive_group(required=False)
verbosity_args.add_argument("-v",
"--verbose",
dest="verbose",
help="Verbose execution.",
action="store_true")
verbosity_args.add_argument("-q",
"--quiet",
dest="quiet",
help="Suppress all output.",
action="store_true")
# FIXME: This is not yet ideal.
# What should we do with tokens like > or ; ?
parser.add_argument(
"link_cmd",
nargs="*",
metavar="<command>",
help=(
"Command to be executed with options and arguments, separated from"
" 'in-toto-run' options by double dash '--'."))
args = parser.parse_args()
log.setLevelVerboseOrQuiet(args.verbose, args.quiet)
# Override defaults in settings.py with environment variables and RCfiles
in_toto.user_settings.set_settings()
# Regular signing and GPG signing are mutually exclusive
if (args.key == None) == (args.gpg == None):
parser.print_usage()
parser.error("Specify either `--key <key path>` or `--gpg [<keyid>]`")
# If `--gpg` was set without argument it has the value `True` and
# we will try to sign with the default key
gpg_use_default = (args.gpg == True)
# Otherwise we interpret it as actual keyid
gpg_keyid = None
if args.gpg != True:
gpg_keyid = args.gpg
# If no_command is specified run in_toto_run without executing a command
if args.no_command:
args.link_cmd = []
elif not args.link_cmd: # pragma: no branch
parser.print_usage()
parser.error("No command specified."
" Please specify (or use the --no-command option)")
try:
# We load the key here because it might prompt the user for a password in
# case the key is encrypted. Something that should not happen in the lib.
key = None
if args.key:
key = util.import_private_key_from_file(args.key, args.key_type)
runlib.in_toto_run(args.step_name, args.materials, args.products,
args.link_cmd, args.record_streams, key, gpg_keyid,
gpg_use_default, args.gpg_home,
args.exclude_patterns, args.base_path,
args.lstrip_paths)
except Exception as e:
log.error("(in-toto-run) {0}: {1}".format(type(e).__name__, e))
sys.exit(1)
sys.exit(0)
def test_in_toto_run_verify_signature(self):
"""Successfully run, verify signed metadata. """
link = in_toto_run(self.step_name, None, None, ["echo", "test"],
self.key, True)
link.verify_signatures({self.key["keyid"]: self.key})
def test_in_toto_run_no_signature(self):
"""Successfully run, verify empty signature field. """
link = in_toto_run(self.step_name, None, None, ["python", "--version"])
self.assertFalse(len(link.signatures))
def test_in_toto_run_verify_signature(self):
"""Successfully run, verify signed metadata. """
link = in_toto_run(self.step_name, None, None, ["python", "--version"],
True, self.key)
link.verify_signature(self.key)
def test_in_toto_run_verify_workdir(self):
"""Successfully run, verify cwd. """
link = in_toto_run(self.step_name, [], [], ["python", "--version"],
record_environment=True)
self.assertEqual(link.signed.environment["workdir"],
os.getcwd().replace("\\", "/"))