def _role_paths_needs_update(role, relative_delegated_paths,
path_hash_prefixes):
# By default, we will assume that the delegator needs no update.
needs_update = False
role_paths = role.get('paths')
# relative_delegated_paths are relative to 'repository'.
# relative_role_paths are relative to 'repository/targets'.
# This is because role_paths are relative to 'repository/targets'.
relative_role_paths = []
for relative_delegated_path in relative_delegated_paths:
assert relative_delegated_path.startswith('targets/')
relative_role_paths.append(relative_delegated_path[8:])
# Check role paths.
if role_paths is None:
logger.warn('No role paths!')
else:
if relative_delegated_paths is not None:
if set(role_paths) == set(relative_role_paths):
logger.debug('Role paths are the same.')
else:
needs_update = True
logger.info('Role paths have changed!')
else:
assert path_hash_prefixes is not None
needs_update = True
logger.info(
'Role paths have been substituted with path_hash_prefixes!')
return needs_update
def get_delegatee_role_from_delegator(delegator_targets_role_name,
relative_delegatee_targets_role_name):
# Extract metadata from the delegator targets role.
delegator_filename = \
os.path.join(METADATA_DIRECTORY,
'{0}.txt'.format(delegator_targets_role_name))
delegator_metadata_dict = \
signerlib.read_metadata_file(delegator_filename)
# TODO: Verify signature on delegator metadata!
delegator_signed_metadata = delegator_metadata_dict['signed']
delegations = delegator_signed_metadata.get('delegations', {})
roles = delegations.get('roles', [])
# Find the delegatee, if it exists, in the delegator.
absolute_delegatee_targets_role_name = \
'{0}/{1}'.format(delegator_targets_role_name,
relative_delegatee_targets_role_name)
role_index = \
signerlib.find_delegated_role(roles, absolute_delegatee_targets_role_name)
role = None
if role_index is None:
logger.info('{0} does not know about {1}'.format(
delegator_targets_role_name, relative_delegatee_targets_role_name))
else:
role = roles[role_index]
return role
def get_delegatee_role_from_delegator(delegator_targets_role_name,
relative_delegatee_targets_role_name):
# Extract metadata from the delegator targets role.
delegator_filename = \
os.path.join(METADATA_DIRECTORY,
'{0}.txt'.format(delegator_targets_role_name))
delegator_metadata_dict = \
signerlib.read_metadata_file(delegator_filename)
# TODO: Verify signature on delegator metadata!
delegator_signed_metadata = delegator_metadata_dict['signed']
delegations = delegator_signed_metadata.get('delegations', {})
roles = delegations.get('roles', [])
# Find the delegatee, if it exists, in the delegator.
absolute_delegatee_targets_role_name = \
'{0}/{1}'.format(delegator_targets_role_name,
relative_delegatee_targets_role_name)
role_index = \
signerlib.find_delegated_role(roles, absolute_delegatee_targets_role_name)
role = None
if role_index is None:
logger.info('{0} does not know about {1}'.format(
delegator_targets_role_name,
relative_delegatee_targets_role_name))
else:
role = roles[role_index]
return role
def _role_paths_needs_update(role, relative_delegated_paths,
path_hash_prefixes):
# By default, we will assume that the delegator needs no update.
needs_update = False
role_paths = role.get('paths')
# relative_delegated_paths are relative to 'repository'.
# relative_role_paths are relative to 'repository/targets'.
# This is because role_paths are relative to 'repository/targets'.
relative_role_paths = []
for relative_delegated_path in relative_delegated_paths:
assert relative_delegated_path.startswith('targets/')
relative_role_paths.append(relative_delegated_path[8:])
# Check role paths.
if role_paths is None:
logger.warn('No role paths!')
else:
if relative_delegated_paths is not None:
if set(role_paths) == set(relative_role_paths):
logger.debug('Role paths are the same.')
else:
needs_update = True
logger.info('Role paths have changed!')
else:
assert path_hash_prefixes is not None
needs_update = True
logger.info('Role paths have been substituted with path_hash_prefixes!')
return needs_update
def update_delegator_metadata(delegator_targets_role_name,
relative_delegatee_targets_role_name,
delegator_targets_role_keys,
delegatee_targets_role_keys,
relative_delegated_paths=None,
path_hash_prefixes=None):
# relative_delegated_paths XOR path_hash_prefixes
assert (relative_delegated_paths is None and path_hash_prefixes is not None) \
or \
(relative_delegated_paths is not None and path_hash_prefixes is None)
if delegator_needs_update(delegator_targets_role_name,
relative_delegatee_targets_role_name,
relative_delegated_paths=relative_delegated_paths,
path_hash_prefixes=path_hash_prefixes):
signercli._update_parent_metadata(METADATA_DIRECTORY,
relative_delegatee_targets_role_name,
delegatee_targets_role_keys,
delegator_targets_role_name,
delegator_targets_role_keys,
delegated_paths=relative_delegated_paths,
path_hash_prefixes=path_hash_prefixes)
else:
logger.info('{0} does not need to be updated about {1}'.format(
delegator_targets_role_name,
relative_delegatee_targets_role_name))
def update_delegator_metadata(delegator_targets_role_name,
relative_delegatee_targets_role_name,
delegator_targets_role_keys,
delegatee_targets_role_keys,
relative_delegated_paths=None,
path_hash_prefixes=None):
# relative_delegated_paths XOR path_hash_prefixes
assert (relative_delegated_paths is None and path_hash_prefixes is not None) \
or \
(relative_delegated_paths is not None and path_hash_prefixes is None)
if delegator_needs_update(
delegator_targets_role_name,
relative_delegatee_targets_role_name,
relative_delegated_paths=relative_delegated_paths,
path_hash_prefixes=path_hash_prefixes):
signercli._update_parent_metadata(
METADATA_DIRECTORY,
relative_delegatee_targets_role_name,
delegatee_targets_role_keys,
delegator_targets_role_name,
delegator_targets_role_keys,
delegated_paths=relative_delegated_paths,
path_hash_prefixes=path_hash_prefixes)
else:
logger.info('{0} does not need to be updated about {1}'.format(
delegator_targets_role_name, relative_delegatee_targets_role_name))
def compress_metadata(metadata_filename):
compressed_metadata_filename = '{0}.gz'.format(metadata_filename)
with open(metadata_filename) as metadata_file, \
gzip.open(compressed_metadata_filename, 'w') as compressed_metadata_file:
shutil.copyfileobj(metadata_file, compressed_metadata_file)
logger.info('Compressed {0} as {1}'.format(metadata_filename,
compressed_metadata_filename))
def compress_metadata(metadata_filename):
compressed_metadata_filename = '{0}.gz'.format(metadata_filename)
with open(metadata_filename) as metadata_file, \
gzip.open(compressed_metadata_filename, 'w') as compressed_metadata_file:
shutil.copyfileobj(metadata_file, compressed_metadata_file)
logger.info('Compressed {0} as {1}'.format(metadata_filename,
compressed_metadata_filename))
def update_release(time_delta, compress=False):
expiration_date = get_expiration_date(time_delta)
release_role_keys = get_keys_for_top_level_role(RELEASE_ROLE_NAME)
version_number = signercli._get_metadata_version(RELEASE_ROLE_FILE)
# Generate and write the signed release metadata.
release_role_filename = \
signerlib.build_release_file(release_role_keys, METADATA_DIRECTORY,
version_number, expiration_date,
compress=compress)
logger.info('Release written to {0}'.format(release_role_filename))
def update_release(time_delta, compress=False):
expiration_date = get_expiration_date(time_delta)
release_role_keys = get_keys_for_top_level_role(RELEASE_ROLE_NAME)
version_number = signercli._get_metadata_version(RELEASE_ROLE_FILE)
# Generate and write the signed release metadata.
release_role_filename = \
signerlib.build_release_file(release_role_keys, METADATA_DIRECTORY,
version_number, expiration_date,
compress=compress)
logger.info('Release written to {0}'.format(release_role_filename))
def update_targets_metadata(targets_role_name,
relative_delegated_paths,
time_delta,
targets_role_keys=None):
# TODO: Ensure that targets_role_name is of the correct form.
assert targets_role_name == 'targets' or \
targets_role_name.startswith('targets/')
# The first time a parent role creates a delegation, a directory containing
# the parent role's name is created in the metadata directory. For example,
# if the targets roles creates a delegated role 'role1', the metadata
# directory would then contain:
# '{METADATA_DIRECTORY}/targets/role1.txt', where 'role1.txt' is the
# delegated role's metadata file.
# If delegated role 'role1' creates its own delegated role 'role2', the
# metadata directory would then contain:
# '{METADATA_DIRECTORY}/targets/role1/role2.txt'.
# When creating a delegated role, if the parent directory already exists,
# this means a prior delegation has been performed by the parent.
parent_role_name = os.path.dirname(targets_role_name)
child_role_name = os.path.basename(targets_role_name)
parent_role_directory = os.path.join(METADATA_DIRECTORY, parent_role_name)
if not os.path.isdir(parent_role_directory):
os.mkdir(parent_role_directory)
# Set the filename of the targets role metadata.
targets_role_filename = os.path.join(parent_role_directory,
'{0}.txt'.format(child_role_name))
expiration_date = get_expiration_date(time_delta)
version_number = signercli._get_metadata_version(targets_role_filename)
# Prepare the targets metadata.
targets_metadata = \
signerlib.generate_targets_metadata(REPOSITORY_DIRECTORY,
relative_delegated_paths,
version_number, expiration_date)
# Where are our signing keys?
if targets_role_keys is None:
logger.info('Looking up keys for {0}'.format(targets_role_name))
targets_role_keys = get_keys_for_targets_role(targets_role_name,
create_missing_keys=True)
else:
logger.info('Using given keys for {0}'.format(targets_role_name))
# Sign and write the targets role metadata.
signercli._sign_and_write_metadata(targets_metadata, targets_role_keys,
targets_role_filename)
def delegator_needs_update(delegator_targets_role_name,
relative_delegatee_targets_role_name,
relative_delegated_paths=None,
path_hash_prefixes=None):
# relative_delegated_paths XOR path_hash_prefixes
assert (relative_delegated_paths is None and path_hash_prefixes is not None) \
or \
(relative_delegated_paths is not None and path_hash_prefixes is None)
# By default, we will assume that the delegator needs no update.
needs_update = False
role = \
get_delegatee_role_from_delegator(delegator_targets_role_name,
relative_delegatee_targets_role_name)
if role is None:
needs_update = True
logger.info('Role needs to be added.')
else:
if _role_path_hash_prefixes_needs_update(role,
relative_delegated_paths,
path_hash_prefixes):
needs_update = True
logger.info('Role needs update.')
elif _role_paths_needs_update(role, relative_delegated_paths,
path_hash_prefixes):
needs_update = True
logger.info('Role needs update.')
else:
logger.info('Role does not need update.')
return needs_update
def update_targets_metadata(targets_role_name, relative_delegated_paths,
time_delta, targets_role_keys=None):
# TODO: Ensure that targets_role_name is of the correct form.
assert targets_role_name == 'targets' or \
targets_role_name.startswith('targets/')
# The first time a parent role creates a delegation, a directory containing
# the parent role's name is created in the metadata directory. For example,
# if the targets roles creates a delegated role 'role1', the metadata
# directory would then contain:
# '{METADATA_DIRECTORY}/targets/role1.txt', where 'role1.txt' is the
# delegated role's metadata file.
# If delegated role 'role1' creates its own delegated role 'role2', the
# metadata directory would then contain:
# '{METADATA_DIRECTORY}/targets/role1/role2.txt'.
# When creating a delegated role, if the parent directory already exists,
# this means a prior delegation has been performed by the parent.
parent_role_name = os.path.dirname(targets_role_name)
child_role_name = os.path.basename(targets_role_name)
parent_role_directory = os.path.join(METADATA_DIRECTORY, parent_role_name)
if not os.path.isdir(parent_role_directory):
os.mkdir(parent_role_directory)
# Set the filename of the targets role metadata.
targets_role_filename = os.path.join(parent_role_directory,
'{0}.txt'.format(child_role_name))
expiration_date = get_expiration_date(time_delta)
version_number = signercli._get_metadata_version(targets_role_filename)
# Prepare the targets metadata.
targets_metadata = \
signerlib.generate_targets_metadata(REPOSITORY_DIRECTORY,
relative_delegated_paths,
version_number, expiration_date)
# Where are our signing keys?
if targets_role_keys is None:
logger.info('Looking up keys for {0}'.format(targets_role_name))
targets_role_keys = get_keys_for_targets_role(targets_role_name,
create_missing_keys=True)
else:
logger.info('Using given keys for {0}'.format(targets_role_name))
# Sign and write the targets role metadata.
signercli._sign_and_write_metadata(targets_metadata, targets_role_keys,
targets_role_filename)
def delegator_needs_update(delegator_targets_role_name,
relative_delegatee_targets_role_name,
relative_delegated_paths=None,
path_hash_prefixes=None):
# relative_delegated_paths XOR path_hash_prefixes
assert (relative_delegated_paths is None and path_hash_prefixes is not None) \
or \
(relative_delegated_paths is not None and path_hash_prefixes is None)
# By default, we will assume that the delegator needs no update.
needs_update = False
role = \
get_delegatee_role_from_delegator(delegator_targets_role_name,
relative_delegatee_targets_role_name)
if role is None:
needs_update = True
logger.info('Role needs to be added.')
else:
if _role_path_hash_prefixes_needs_update(role, relative_delegated_paths,
path_hash_prefixes):
needs_update = True
logger.info('Role needs update.')
elif _role_paths_needs_update(role, relative_delegated_paths,
path_hash_prefixes):
needs_update = True
logger.info('Role needs update.')
else:
logger.info('Role does not need update.')
return needs_update
def get_keys_for_targets_role(targets_role_name, create_missing_keys=False):
"""Given the name of a targets role (targets_role_name) and its list of
passwords (targets_role_passwords), return the RSA key IDs for a targets role.
Side effect: Load the aforementioned RSA keys into the global TUF
keystore."""
# Get the list of passwords for this role.
targets_role_passwords = ROLE_NAME_TO_PASSWORDS[targets_role_name]
if targets_role_name == 'targets':
# Ask root about targets' keys.
root_metadata_dict = signerlib.read_metadata_file(ROOT_ROLE_FILE)
# TODO: Verify signature on root metadata!
signed_root_metadata = root_metadata_dict['signed']
targets_role_keys = signed_root_metadata['roles']['targets']['keyids']
else:
# Ask the targets role itself.
targets_role_filename = os.path.join(
METADATA_DIRECTORY, '{0}.txt'.format(targets_role_name))
# If the targets role exists...
if os.path.exists(targets_role_filename):
targets_role_metadata_dict = \
signerlib.read_metadata_file(targets_role_filename)
# TODO: Verify signature on targets role metadata!
targets_role_metadata_signatures = \
targets_role_metadata_dict['signatures']
targets_role_keys = []
# Assume keys are listed in the same order as passwords.
for targets_role_metadata_signature in targets_role_metadata_signatures:
targets_role_keys.append(
targets_role_metadata_signature['keyid'])
# Assume that there are as many keys as there are passwords.
assert len(targets_role_keys) == len(targets_role_passwords)
# Otherwise...
else:
if create_missing_keys:
logger.info('Creating keys for {0}'.format(targets_role_name))
targets_role_keys = generate_rsa_keys(targets_role_passwords)
else:
raise MissingKeys(targets_role_name)
# Decrypt and load the keys of the targets role.
loaded_targets_role_keys = \
keystore.load_keystore_from_keyfiles(KEYSTORE_DIRECTORY, targets_role_keys,
targets_role_passwords)
assert targets_role_keys == loaded_targets_role_keys
return targets_role_keys
def get_keys_for_targets_role(targets_role_name, create_missing_keys=False):
"""Given the name of a targets role (targets_role_name) and its list of
passwords (targets_role_passwords), return the RSA key IDs for a targets role.
Side effect: Load the aforementioned RSA keys into the global TUF
keystore."""
# Get the list of passwords for this role.
targets_role_passwords = ROLE_NAME_TO_PASSWORDS[targets_role_name]
if targets_role_name == 'targets':
# Ask root about targets' keys.
root_metadata_dict = signerlib.read_metadata_file(ROOT_ROLE_FILE)
# TODO: Verify signature on root metadata!
signed_root_metadata = root_metadata_dict['signed']
targets_role_keys = signed_root_metadata['roles']['targets']['keyids']
else:
# Ask the targets role itself.
targets_role_filename = os.path.join(METADATA_DIRECTORY,
'{0}.txt'.format(targets_role_name))
# If the targets role exists...
if os.path.exists(targets_role_filename):
targets_role_metadata_dict = \
signerlib.read_metadata_file(targets_role_filename)
# TODO: Verify signature on targets role metadata!
targets_role_metadata_signatures = \
targets_role_metadata_dict['signatures']
targets_role_keys = []
# Assume keys are listed in the same order as passwords.
for targets_role_metadata_signature in targets_role_metadata_signatures:
targets_role_keys.append(targets_role_metadata_signature['keyid'])
# Assume that there are as many keys as there are passwords.
assert len(targets_role_keys) == len(targets_role_passwords)
# Otherwise...
else:
if create_missing_keys:
logger.info('Creating keys for {0}'.format(targets_role_name))
targets_role_keys = generate_rsa_keys(targets_role_passwords)
else:
raise MissingKeys(targets_role_name)
# Decrypt and load the keys of the targets role.
loaded_targets_role_keys = \
keystore.load_keystore_from_keyfiles(KEYSTORE_DIRECTORY, targets_role_keys,
targets_role_passwords)
assert targets_role_keys == loaded_targets_role_keys
return targets_role_keys
def _role_path_hash_prefixes_needs_update(role, relative_delegated_paths,
path_hash_prefixes):
# By default, we will assume that the delegator needs no update.
needs_update = False
role_path_hash_prefixes = role.get('path_hash_prefixes')
# Otherwise, check role path_hash_prefixes.
if role_path_hash_prefixes is None:
logger.warn('No role path_hash_prefixes!')
else:
if path_hash_prefixes is not None:
if set(role_path_hash_prefixes) == set(path_hash_prefixes):
logger.debug('Role path_hash_prefixes is the same.')
else:
needs_update = True
logger.debug('Role path_hash_prefixes has changed!')
else:
assert relative_delegated_paths is not None
needs_update = True
logger.info('Role path_hash_prefixes has been substituted with paths!')
return needs_update
def _role_path_hash_prefixes_needs_update(role, relative_delegated_paths,
path_hash_prefixes):
# By default, we will assume that the delegator needs no update.
needs_update = False
role_path_hash_prefixes = role.get('path_hash_prefixes')
# Otherwise, check role path_hash_prefixes.
if role_path_hash_prefixes is None:
logger.warn('No role path_hash_prefixes!')
else:
if path_hash_prefixes is not None:
if set(role_path_hash_prefixes) == set(path_hash_prefixes):
logger.debug('Role path_hash_prefixes is the same.')
else:
needs_update = True
logger.debug('Role path_hash_prefixes has changed!')
else:
assert relative_delegated_paths is not None
needs_update = True
logger.info(
'Role path_hash_prefixes has been substituted with paths!')
return needs_update
def metadata_matches_data(metadata_directory, targets_directory, full_role_name,
files_directory, recursive_walk=False,
followlinks=True,
file_predicate=signerlib.accept_any_file):
"""
Return True if metadata matches data for the target role; False otherwise.
"""
# Assume that metadata lives in a file specified by the full role name.
metadata_filename = full_role_name + ".txt"
metadata_filename = os.path.join(metadata_directory, metadata_filename)
try:
metadata_file = open(metadata_filename)
except:
raise MissingTargetMetadataError(metadata_filename)
else:
all_metadata = json.load(metadata_file)
metadata_file.close()
# TODO: Use TUF to verify that all_metadata is correctly signed.
signed_metadata = all_metadata["signed"]
# Check that the metadata is well-formed.
signed_metadata = tuf.formats.TargetsFile.from_metadata(signed_metadata)
expected_targets = signed_metadata.info["targets"]
# We begin by assuming that everything is all right.
matched = True
# For expected_file in metadata, does it match the observed_file in targets?
for expected_file in expected_targets:
observed_file = os.path.join(targets_directory, expected_file)
if os.path.exists(observed_file):
# Does expected_file describe observed_file?
expected_file_metadata = expected_targets[expected_file]
# Compare every hash of the expected file with the equivalent hash of
# the observed file.
expected_file_hashes = expected_file_metadata["hashes"]
for hash_algorithm, expected_file_digest in expected_file_hashes.iteritems():
observed_file_digest_object = \
tuf.hash.digest_filename(observed_file, algorithm=hash_algorithm)
observed_file_digest = observed_file_digest_object.hexdigest()
if observed_file_digest != expected_file_digest:
# Metadata has probably diverged from data.
logger.info("{0} != {1}".format(observed_file, expected_file))
matched = False
break
# Break out of the outer for loop in case we found a mismatch.
if not matched: break
else:
# expected_file was deleted, so metadata has diverged from data.
logger.info("{0} has been deleted".format(expected_file))
matched = False
break
# For observed_file in targets, does it match the expected_file in metadata?
if matched:
# FIXME: Temporary hack to workaround Python 2 not returning filenames in
# Unicode unless you do tricks like this:
# http://docs.python.org/2/howto/unicode.html#unicode-filenames
files_directory = unicode(files_directory, encoding="utf-8")
# Get the list of observed target files.
observed_targets = signerlib.get_targets(files_directory,
recursive_walk=recursive_walk,
followlinks=followlinks,
file_predicate=file_predicate)
for observed_file in observed_targets:
# Ensure that form of observed_file conforms to that of expected_file.
# Presently, this means that they do not share the "targets/" prefix.
assert observed_file.startswith(targets_directory)
observed_file = observed_file[len(targets_directory)+1:]
# observed_file was added, so metadata has diverged from data.
if observed_file not in expected_targets:
logger.info("{0} is new".format(observed_file))
matched = False
break
return matched
def update_unclaimed_targets():
# For simplicity, ensure that we can evenly distribute MAX_NUMBER_OF_BINS
# over NUMBER_OF_BINS.
assert MAX_NUMBER_OF_BINS % NUMBER_OF_BINS == 0
# Get all possible targets.
absolute_delegated_paths = signerlib.get_targets(
delegate.TARGETS_DIRECTORY, recursive_walk=True, followlinks=True)
logger.info('There are {0} total PyPI targets.'.format(
len(absolute_delegated_paths)))
# Record the absolute delegated paths that fall into each bin.
absolute_delegated_paths_in_bin = \
{bin_index: [] for bin_index in xrange(MAX_NUMBER_OF_BINS)}
# Assign every path to its bin.
for absolute_delegated_path in absolute_delegated_paths:
assert absolute_delegated_path.startswith(delegate.TARGETS_DIRECTORY +
'/')
relative_delegated_path = \
absolute_delegated_path[len(delegate.TARGETS_DIRECTORY)+1:]
relative_delegated_path_digest = hasher.digest(algorithm=HASH_FUNCTION)
relative_delegated_path_digest.update(relative_delegated_path)
relative_delegated_path_hash = relative_delegated_path_digest.hexdigest(
)
relative_delegated_path_hash_prefix = \
relative_delegated_path_hash[:PREFIX_LENGTH]
# Convert a base-16 (hex) number to a base-10 (dec) number.
bin_index = int(relative_delegated_path_hash_prefix, 16)
assert bin_index > -1
assert bin_index < MAX_NUMBER_OF_BINS
absolute_delegated_paths_in_bin[bin_index] += [absolute_delegated_path]
# Delegate every target to the "unclaimed" targets role.
unclaimed_relative_delegated_paths = ["targets/gems/"]
# TODO: Update delegatee only if necessary to do so.
unclaimed_relative_target_paths = []
delegate.update_targets_metadata(delegate.UNCLAIMED_TARGETS_ROLE_NAME,
unclaimed_relative_target_paths,
datetime.timedelta(days=365))
# TODO: Comment on shared keys, and why this must come after
# update_targets_metadata.
unclaimed_targets_role_keys = \
delegate.get_keys_for_targets_role(delegate.UNCLAIMED_TARGETS_ROLE_NAME)
delegate.make_delegation(
delegate.TARGETS_ROLE_NAME,
delegate.UNCLAIMED_TARGETS_ROLE_NAME,
relative_delegated_paths=unclaimed_relative_delegated_paths)
# Delegate from the "unclaimed" targets role to each bin.
bin_offset = MAX_NUMBER_OF_BINS // NUMBER_OF_BINS
for global_bin_index in xrange(0, MAX_NUMBER_OF_BINS, bin_offset):
# The bin index in hex padded from the left with zeroes for up to the
# PREFIX_LENGTH.
global_bin_index_in_hex = hex(global_bin_index)[2:].zfill(
PREFIX_LENGTH)
relative_binned_targets_role_name = global_bin_index_in_hex
absolute_binned_targets_role_name = \
os.path.join(delegate.UNCLAIMED_TARGETS_ROLE_NAME,
relative_binned_targets_role_name)
absolute_delegated_paths_in_this_bin = []
path_hash_prefixes = []
for local_bin_index in xrange(global_bin_index,
global_bin_index + bin_offset):
absolute_delegated_paths_in_this_bin.extend(
absolute_delegated_paths_in_bin[local_bin_index])
local_bin_index_in_hex = hex(local_bin_index)[2:].zfill(
PREFIX_LENGTH)
path_hash_prefixes.append(local_bin_index_in_hex)
relative_delegated_paths_in_this_bin = \
delegate.get_relative_delegated_paths(absolute_delegated_paths_in_this_bin)
# TODO: Update delegator only if necessary to do so.
delegate.update_delegator_metadata(
delegate.UNCLAIMED_TARGETS_ROLE_NAME,
relative_binned_targets_role_name,
unclaimed_targets_role_keys,
unclaimed_targets_role_keys,
path_hash_prefixes=path_hash_prefixes)
logger.info('Delegated from {0} to {1}'.format(
delegate.UNCLAIMED_TARGETS_ROLE_NAME,
absolute_binned_targets_role_name))
# TODO: Update delegatee only if necessary to do so.
delegate.update_targets_metadata(
absolute_binned_targets_role_name,
relative_delegated_paths_in_this_bin,
datetime.timedelta(days=90),
targets_role_keys=unclaimed_targets_role_keys)
logger.info('Wrote {0}'.format(absolute_binned_targets_role_name))
# Compress "unclaimed" targets role metadata.
unclaimed_targets_role_filename = \
os.path.join(delegate.METADATA_DIRECTORY,
'{0}.txt'.format(delegate.UNCLAIMED_TARGETS_ROLE_NAME))
delegate.compress_metadata(unclaimed_targets_role_filename)
# Compute statistics and check sanity.
expected_number_of_paths_per_bin = \
len(absolute_delegated_paths)/MAX_NUMBER_OF_BINS
observed_number_of_paths_in_all_bins = \
sum((len(paths) for paths in absolute_delegated_paths_in_bin.values()))
observed_number_of_bins = len(absolute_delegated_paths_in_bin)
observed_number_of_paths_per_bin = \
observed_number_of_paths_in_all_bins/observed_number_of_bins
# Each of the delegated paths must have been assigned to a bin.
assert observed_number_of_paths_in_all_bins == len(
absolute_delegated_paths)
# The observed number of bins must be equal to the expected number of bins.
assert observed_number_of_bins == MAX_NUMBER_OF_BINS
logger.info('Expected number of paths per bin: {0}'.format(
expected_number_of_paths_per_bin))
logger.info('Observed number of paths per bin: {0}'.format(
observed_number_of_paths_per_bin))
def update_unclaimed_targets():
# For simplicity, ensure that we can evenly distribute MAX_NUMBER_OF_BINS
# over NUMBER_OF_BINS.
assert MAX_NUMBER_OF_BINS % NUMBER_OF_BINS == 0
# Get all possible targets.
absolute_delegated_paths = signerlib.get_targets(delegate.TARGETS_DIRECTORY,
recursive_walk=True,
followlinks=True)
logger.info('There are {0} total PyPI targets.'.format(len(
absolute_delegated_paths)))
# Record the absolute delegated paths that fall into each bin.
absolute_delegated_paths_in_bin = \
{bin_index: [] for bin_index in xrange(MAX_NUMBER_OF_BINS)}
# Assign every path to its bin.
for absolute_delegated_path in absolute_delegated_paths:
assert absolute_delegated_path.startswith(delegate.TARGETS_DIRECTORY+'/')
relative_delegated_path = \
absolute_delegated_path[len(delegate.TARGETS_DIRECTORY)+1:]
relative_delegated_path_digest = hasher.digest(algorithm=HASH_FUNCTION)
relative_delegated_path_digest.update(relative_delegated_path)
relative_delegated_path_hash = relative_delegated_path_digest.hexdigest()
relative_delegated_path_hash_prefix = \
relative_delegated_path_hash[:PREFIX_LENGTH]
# Convert a base-16 (hex) number to a base-10 (dec) number.
bin_index = int(relative_delegated_path_hash_prefix, 16)
assert bin_index > -1
assert bin_index < MAX_NUMBER_OF_BINS
absolute_delegated_paths_in_bin[bin_index] += [absolute_delegated_path]
# Delegate every target to the "unclaimed" targets role.
# Presently, every target falls under the "simple/" and "packages/"
# directories.
unclaimed_relative_delegated_paths = ["targets/simple/", "targets/packages/"]
# TODO: Update delegatee only if necessary to do so.
unclaimed_relative_target_paths = []
delegate.update_targets_metadata(delegate.UNCLAIMED_TARGETS_ROLE_NAME,
unclaimed_relative_target_paths,
datetime.timedelta(days=365))
# TODO: Comment on shared keys, and why this must come after
# update_targets_metadata.
unclaimed_targets_role_keys = \
delegate.get_keys_for_targets_role(delegate.UNCLAIMED_TARGETS_ROLE_NAME)
delegate.make_delegation(delegate.TARGETS_ROLE_NAME,
delegate.UNCLAIMED_TARGETS_ROLE_NAME,
relative_delegated_paths=unclaimed_relative_delegated_paths)
# Delegate from the "unclaimed" targets role to each bin.
bin_offset = MAX_NUMBER_OF_BINS // NUMBER_OF_BINS
for global_bin_index in xrange(0, MAX_NUMBER_OF_BINS, bin_offset):
# The bin index in hex padded from the left with zeroes for up to the
# PREFIX_LENGTH.
global_bin_index_in_hex = hex(global_bin_index)[2:].zfill(PREFIX_LENGTH)
relative_binned_targets_role_name = global_bin_index_in_hex
absolute_binned_targets_role_name = \
os.path.join(delegate.UNCLAIMED_TARGETS_ROLE_NAME,
relative_binned_targets_role_name)
absolute_delegated_paths_in_this_bin = []
path_hash_prefixes = []
for local_bin_index in xrange(global_bin_index,
global_bin_index+bin_offset):
absolute_delegated_paths_in_this_bin.extend(
absolute_delegated_paths_in_bin[local_bin_index])
local_bin_index_in_hex = hex(local_bin_index)[2:].zfill(PREFIX_LENGTH)
path_hash_prefixes.append(local_bin_index_in_hex)
relative_delegated_paths_in_this_bin = \
delegate.get_relative_delegated_paths(absolute_delegated_paths_in_this_bin)
# TODO: Update delegator only if necessary to do so.
delegate.update_delegator_metadata(delegate.UNCLAIMED_TARGETS_ROLE_NAME,
relative_binned_targets_role_name,
unclaimed_targets_role_keys,
unclaimed_targets_role_keys,
path_hash_prefixes=path_hash_prefixes)
logger.info('Delegated from {0} to {1}'.format(
delegate.UNCLAIMED_TARGETS_ROLE_NAME,
absolute_binned_targets_role_name))
# TODO: Update delegatee only if necessary to do so.
delegate.update_targets_metadata(absolute_binned_targets_role_name,
relative_delegated_paths_in_this_bin,
datetime.timedelta(days=90),
targets_role_keys=unclaimed_targets_role_keys)
logger.info('Wrote {0}'.format(absolute_binned_targets_role_name))
# Compress "unclaimed" targets role metadata.
unclaimed_targets_role_filename = \
os.path.join(delegate.METADATA_DIRECTORY,
'{0}.txt'.format(delegate.UNCLAIMED_TARGETS_ROLE_NAME))
delegate.compress_metadata(unclaimed_targets_role_filename)
# Compute statistics and check sanity.
expected_number_of_paths_per_bin = \
len(absolute_delegated_paths)/MAX_NUMBER_OF_BINS
observed_number_of_paths_in_all_bins = \
sum((len(paths) for paths in absolute_delegated_paths_in_bin.values()))
observed_number_of_bins = len(absolute_delegated_paths_in_bin)
observed_number_of_paths_per_bin = \
observed_number_of_paths_in_all_bins/observed_number_of_bins
# Each of the delegated paths must have been assigned to a bin.
assert observed_number_of_paths_in_all_bins == len(absolute_delegated_paths)
# The observed number of bins must be equal to the expected number of bins.
assert observed_number_of_bins == MAX_NUMBER_OF_BINS
logger.info('Expected number of paths per bin: {0}'.format(
expected_number_of_paths_per_bin))
logger.info('Observed number of paths per bin: {0}'.format(
observed_number_of_paths_per_bin))