def iterator(coefficient_fn: COEFFICIENT_FN, model_fn: MODEL_FN,
client_state_fn: CLIENT_STATE_FN,
server_optimizer_fn: OPTIMIZER_FN,
client_optimizer_fn: OPTIMIZER_FN):
model = model_fn()
client_state = client_state_fn()
init_tf = tff.tf_computation(
lambda: __initialize_server(model_fn, server_optimizer_fn))
server_state_type = init_tf.type_signature.result
client_state_type = tff.framework.type_from_tensors(client_state)
update_server_tf = tff.tf_computation(
lambda state, weights_delta: __update_server(
state, weights_delta, model_fn, server_optimizer_fn,
tf.function(server.update)),
(server_state_type, server_state_type.model.trainable))
state_to_message_tf = tff.tf_computation(
lambda state: __state_to_message(state, tf.function(server.to_message)
), server_state_type)
dataset_type = tff.SequenceType(model.input_spec)
server_message_type = state_to_message_tf.type_signature.result
update_client_tf = tff.tf_computation(
lambda dataset, state, message: __update_client(
dataset, state, message, coefficient_fn, model_fn,
client_optimizer_fn, tf.function(client.update)),
(dataset_type, client_state_type, server_message_type))
federated_server_state_type = tff.type_at_server(server_state_type)
federated_dataset_type = tff.type_at_clients(dataset_type)
federated_client_state_type = tff.type_at_clients(client_state_type)
def init_tff():
return tff.federated_value(init_tf(), tff.SERVER)
def next_tff(server_state, datasets, client_states):
message = tff.federated_map(state_to_message_tf, server_state)
broadcast = tff.federated_broadcast(message)
outputs = tff.federated_map(update_client_tf,
(datasets, client_states, broadcast))
weights_delta = tff.federated_mean(outputs.weights_delta,
weight=outputs.client_weight)
metrics = model.federated_output_computation(outputs.metrics)
next_state = tff.federated_map(update_server_tf,
(server_state, weights_delta))
return next_state, metrics, outputs.client_state
return tff.templates.IterativeProcess(
initialize_fn=tff.federated_computation(init_tff),
next_fn=tff.federated_computation(
next_tff, (federated_server_state_type, federated_dataset_type,
federated_client_state_type)))
def local_acc(model, all_batches):
acc = tff.sequence_sum(
tff.sequence_map(
tff.federated_computation(lambda b: batch_pre(model, b),
BATCH_TYPE), all_batches))
acc_l = tff.sequence_map(
tff.federated_computation(lambda b: batch_pre(model, b), BATCH_TYPE),
all_batches)
return acc
def evaluator(coefficient_fn: COEFFICIENT_FN, model_fn: MODEL_FN,
client_state_fn: CLIENT_STATE_FN):
model = model_fn()
client_state = client_state_fn()
dataset_type = tff.SequenceType(model.input_spec)
client_state_type = tff.framework.type_from_tensors(client_state)
weights_type = tff.framework.type_from_tensors(
tff.learning.ModelWeights.from_model(model))
evaluate_client_tf = tff.tf_computation(
lambda dataset, state, weights: __evaluate_client(
dataset, state, weights, coefficient_fn, model_fn,
tf.function(client.evaluate)),
(dataset_type, client_state_type, weights_type))
federated_weights_type = tff.type_at_server(weights_type)
federated_dataset_type = tff.type_at_clients(dataset_type)
federated_client_state_type = tff.type_at_clients(client_state_type)
def evaluate(weights, datasets, client_states):
broadcast = tff.federated_broadcast(weights)
outputs = tff.federated_map(evaluate_client_tf,
(datasets, client_states, broadcast))
confusion_matrix = tff.federated_sum(outputs.confusion_matrix)
aggregated_metrics = model.federated_output_computation(
outputs.metrics)
collected_metrics = tff.federated_collect(outputs.metrics)
return confusion_matrix, aggregated_metrics, collected_metrics
return tff.federated_computation(
evaluate, (federated_weights_type, federated_dataset_type,
federated_client_state_type))
def validator(
model_fn: MODEL_FN,
client_state_fn: CLIENT_STATE_FN
):
model = model_fn()
client_state = client_state_fn()
dataset_type = tff.SequenceType(model.input_spec)
client_state_type = tff.framework.type_from_tensors(client_state)
validate_client_tf = tff.tf_computation(
lambda dataset, state: __validate_client(
dataset,
state,
model_fn,
tf.function(client.validate)
),
(dataset_type, client_state_type)
)
federated_dataset_type = tff.type_at_clients(dataset_type)
federated_client_state_type = tff.type_at_clients(client_state_type)
def validate(datasets, client_states):
outputs = tff.federated_map(validate_client_tf, (datasets, client_states))
metrics = model.federated_output_computation(outputs.metrics)
return metrics
return tff.federated_computation(
validate,
(federated_dataset_type, federated_client_state_type)
)
def validator(coefficient_fn: COEFFICIENT_FN, model_fn: MODEL_FN,
client_state_fn: CLIENT_STATE_FN):
model = model_fn()
client_state = client_state_fn()
dataset_type = tff.SequenceType(model.input_spec)
client_state_type = tff.framework.type_from_tensors(client_state)
weights_type = tff.learning.framework.weights_type_from_model(model)
validate_client_tf = tff.tf_computation(
lambda dataset, state, weights: __validate_client(
dataset, state, weights, coefficient_fn, model_fn,
tf.function(client.validate)),
(dataset_type, client_state_type, weights_type))
federated_weights_type = tff.type_at_server(weights_type)
federated_dataset_type = tff.type_at_clients(dataset_type)
federated_client_state_type = tff.type_at_clients(client_state_type)
def validate(weights, datasets, client_states):
broadcast = tff.federated_broadcast(weights)
outputs = tff.federated_map(validate_client_tf,
(datasets, client_states, broadcast))
metrics = model.federated_output_computation(outputs.metrics)
return metrics
return tff.federated_computation(
validate, (federated_weights_type, federated_dataset_type,
federated_client_state_type))
def federated_output_computation_from_metrics(
metrics: List[tf.keras.metrics.Metric]) -> tff.federated_computation:
"""Produces a federated computation for aggregating Keras metrics.
This can be used to evaluate both Keras and non-Keras models using Keras
metrics. Aggregates metrics across clients by summing their internal
variables, producing new metrics with summed internal variables, and calling
metric.result() on each. See `federated_aggregate_keras_metric` for details.
Args:
metrics: A List of `tf.keras.metrics.Metric` to aggregate.
Returns:
A `tff.federated_computation` aggregating metrics across clients by summing
their internal variables, producing new metrics with summed internal
variables, and calling metric.result() on each.
"""
# Get a sample of metric variables to use to determine its type.
sample_metric_variables = read_metric_variables(metrics)
metric_variable_type_dict = tf.nest.map_structure(
tf.TensorSpec.from_tensor, sample_metric_variables)
federated_local_outputs_type = tff.type_at_clients(
metric_variable_type_dict)
def federated_output(local_outputs):
return federated_aggregate_keras_metric(metrics, local_outputs)
federated_output_computation = tff.federated_computation(
federated_output, federated_local_outputs_type)
return federated_output_computation
def local_eval(model, all_batches):
#
return tff.sequence_sum(
tff.sequence_map(
tff.federated_computation(
lambda b: batch_loss(model, b), BATCH_TYPE),
all_batches))
def iterator(
model_fn: MODEL_FN,
client_state_fn: CLIENT_STATE_FN,
client_optimizer_fn: OPTIMIZER_FN
):
model = model_fn()
client_state = client_state_fn()
init_tf = tff.tf_computation(
lambda: ()
)
server_state_type = init_tf.type_signature.result
client_state_type = tff.framework.type_from_tensors(client_state)
dataset_type = tff.SequenceType(model.input_spec)
update_client_tf = tff.tf_computation(
lambda dataset, state: __update_client(
dataset,
state,
model_fn,
client_optimizer_fn,
tf.function(client.update)
),
(dataset_type, client_state_type)
)
federated_server_state_type = tff.type_at_server(server_state_type)
federated_dataset_type = tff.type_at_clients(dataset_type)
federated_client_state_type = tff.type_at_clients(client_state_type)
def init_tff():
return tff.federated_value(init_tf(), tff.SERVER)
def next_tff(server_state, datasets, client_states):
outputs = tff.federated_map(update_client_tf, (datasets, client_states))
metrics = model.federated_output_computation(outputs.metrics)
return server_state, metrics, outputs.client_state
return tff.templates.IterativeProcess(
initialize_fn=tff.federated_computation(init_tff),
next_fn=tff.federated_computation(
next_tff,
(federated_server_state_type, federated_dataset_type, federated_client_state_type)
)
)
def local_eval(model, all_batches):
# tff.sequence_sum Replace with `tff.sequence_average()` once implemented.
return tff.sequence_sum(
tff.sequence_map(
#？？？？？
tff.federated_computation(lambda b: batch_loss(model, b),
BATCH_TYPE),
all_batches))
def local_eval(model, all_batchs):
#计算平均loss值时使用tff.sequence_average函数
return tff.sequence_sum(
#序列map函数，将all_batchs中的每个batch都进行loss值计算
#tff.sequence_map与tff.sequence_reduce的差别在于map是并行计算，reduce是串行计算
tff.sequence_map(
tff.federated_computation(lambda b: batch_loss(model, b),
BATCH_TYPE), all_batchs))
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
channel = grpc.insecure_channel('{}:{}'.format(FLAGS.host, FLAGS.port))
context = tff.backends.native.create_remote_python_execution_context(
channels=[channel])
tff.framework.set_default_context(context)
print(tff.federated_computation(lambda: 'Hello World')())
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
channel = grpc.insecure_channel('{}:{}'.format(FLAGS.host, FLAGS.port))
remote_executor = tff.framework.RemoteExecutor(channel)
caching_executor = tff.framework.CachingExecutor(remote_executor)
lambda_executor = tff.framework.LambdaExecutor(caching_executor)
tff.framework.set_default_executor(lambda_executor)
print(tff.federated_computation(lambda: 'Hello World')())
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
channel = grpc.insecure_channel('{}:{}'.format(FLAGS.host, FLAGS.port))
ex = tff.framework.RemoteExecutor(channel)
ex = tff.framework.CachingExecutor(ex)
ex = tff.framework.ReferenceResolvingExecutor(ex)
factory = tff.framework.create_executor_factory(lambda _: ex)
context = tff.framework.ExecutionContext(factory)
tff.framework.set_default_context(context)
print(tff.federated_computation(lambda: 'Hello World')())
def build_federated_averaging_process_attacked(
model_fn,
client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.1),
server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=1.0),
stateful_delta_aggregate_fn=build_stateless_mean(),
client_update_tf=ClientExplicitBoosting(boost_factor=1.0)):
"""Builds the TFF computations for optimization using federated averaging with potentially malicious clients.
Args:
model_fn: A no-arg function that returns a `tff.learning.Model`.
client_optimizer_fn: A no-arg function that returns a
`tf.keras.optimizers.Optimizer`, use during local client training.
server_optimizer_fn: A no-arg function that returns a
`tf.keras.optimizers.Optimizer`, use to apply updates to the global model.
stateful_delta_aggregate_fn: A 'tff.computation' that aggregates model
deltas placed@CLIENTS to an aggregated model delta placed@SERVER.
client_update_tf: a 'tf.function' computes the ClientOutput.
Returns:
A `tff.templates.IterativeProcess`.
"""
dummy_model_for_metadata = model_fn()
server_init_tf = build_server_init_fn(
model_fn, server_optimizer_fn,
stateful_delta_aggregate_fn.initialize())
server_state_type = server_init_tf.type_signature.result
server_update_fn = build_server_update_fn(model_fn, server_optimizer_fn,
server_state_type,
server_state_type.model)
tf_dataset_type = tff.SequenceType(dummy_model_for_metadata.input_spec)
client_update_fn = build_client_update_fn(model_fn, client_optimizer_fn,
client_update_tf,
tf_dataset_type,
server_state_type.model)
federated_server_state_type = tff.FederatedType(server_state_type,
tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
run_one_round_tff = build_run_one_round_fn_attacked(
server_update_fn, client_update_fn, stateful_delta_aggregate_fn,
dummy_model_for_metadata, federated_server_state_type,
federated_dataset_type)
return tff.templates.IterativeProcess(
initialize_fn=tff.federated_computation(
lambda: tff.federated_value(server_init_tf(), tff.SERVER)),
next_fn=run_one_round_tff)
def build_federated_averaging_process(
model_fn,
client_optimizer_fn,
server_optimizer_fn=lambda: flars_optimizer.FLARSOptimizer(learning_rate=
1.0)):
"""Builds the TFF computations for optimization using federated averaging.
Args:
model_fn: A no-arg function that returns a `tff.learning.Model`.
client_optimizer_fn: A no-arg function that returns a
`tf.keras.optimizers.Optimizer` for the local client training.
server_optimizer_fn: A no-arg function that returns a
`tf.keras.optimizers.Optimizer` for applying updates on the server.
Returns:
A `tff.templates.IterativeProcess`.
"""
with tf.Graph().as_default():
dummy_model_for_metadata = model_fn()
type_signature_grads_norm = tuple(
weight.dtype for weight in tf.nest.flatten(
dummy_model_for_metadata.trainable_variables))
server_init_tf = build_server_init_fn(model_fn, server_optimizer_fn)
server_state_type = server_init_tf.type_signature.result
server_update_fn = build_server_update_fn(model_fn, server_optimizer_fn,
server_state_type,
server_state_type.model,
type_signature_grads_norm)
tf_dataset_type = tff.SequenceType(dummy_model_for_metadata.input_spec)
client_update_fn = build_client_update_fn(model_fn, client_optimizer_fn,
tf_dataset_type,
server_state_type.model)
federated_server_state_type = tff.FederatedType(server_state_type,
tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
run_one_round_tff = build_run_one_round_fn(server_update_fn,
client_update_fn,
dummy_model_for_metadata,
federated_server_state_type,
federated_dataset_type)
return tff.templates.IterativeProcess(
initialize_fn=tff.federated_computation(
lambda: tff.federated_eval(server_init_tf, tff.SERVER)),
next_fn=run_one_round_tff)
def build_federated_averaging_process(
model_fn,
server_optimizer_fn=lambda: flars_optimizer.FLARSOptimizer(learning_rate=
1.0)):
"""Builds the TFF computations for optimization using federated averaging.
Args:
model_fn: A no-arg function that returns a `tff.learning.TrainableModel`.
server_optimizer_fn: A no-arg function that returns a
`tf.keras.optimizers.Optimizer`.
Returns:
A `tff.utils.IterativeProcess`.
"""
dummy_model_for_metadata = model_fn()
type_signature_grads_norm = tff.NamedTupleType([
weight.dtype for weight in tf.nest.flatten(
_get_weights(dummy_model_for_metadata).trainable)
])
server_init_tf = build_server_init_fn(model_fn, server_optimizer_fn)
server_state_type = server_init_tf.type_signature.result
server_update_fn = build_server_update_fn(model_fn, server_optimizer_fn,
server_state_type,
server_state_type.model,
type_signature_grads_norm)
tf_dataset_type = tff.SequenceType(dummy_model_for_metadata.input_spec)
client_update_fn = build_client_update_fn(model_fn, tf_dataset_type,
server_state_type.model)
federated_server_state_type = tff.FederatedType(server_state_type,
tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
run_one_round_tff = build_run_one_round_fn(server_update_fn,
client_update_fn,
dummy_model_for_metadata,
federated_server_state_type,
federated_dataset_type)
return tff.utils.IterativeProcess(initialize_fn=tff.federated_computation(
lambda: tff.federated_value(server_init_tf(), tff.SERVER)),
next_fn=run_one_round_tff)
def evaluator(
model_fn: MODEL_FN,
client_state_fn: CLIENT_STATE_FN
):
model = model_fn()
client_state = client_state_fn()
dataset_type = tff.SequenceType(model.input_spec)
client_state_type = tff.framework.type_from_tensors(client_state)
evaluate_client_tf = tff.tf_computation(
lambda dataset, state: __evaluate_client(
dataset,
state,
model_fn,
tf.function(client.evaluate)
),
(dataset_type, client_state_type)
)
federated_dataset_type = tff.type_at_clients(dataset_type)
federated_client_state_type = tff.type_at_clients(client_state_type)
def evaluate(datasets, client_states):
outputs = tff.federated_map(evaluate_client_tf, (datasets, client_states))
confusion_matrix = tff.federated_sum(outputs.confusion_matrix)
aggregated_metrics = model.federated_output_computation(outputs.metrics)
collected_metrics = tff.federated_collect(outputs.metrics)
return confusion_matrix, aggregated_metrics, collected_metrics
return tff.federated_computation(
evaluate,
(federated_dataset_type, federated_client_state_type)
)
NUM_CLIENTS = 10
# NOTE: If the statement below fails, it means that you are
# using an older version of TFF without the high-performance
# executor stack. Call `tff.framework.set_default_executor()`
# instead to use the default reference runtime.
#if six.PY3:
# tff.framework.set_default_executor(
# tff.framework.create_local_executor(NUM_CLIENTS))
# tff.framework.set_default_executor()
if six.PY3:
tff.framework.set_default_executor()
# tff.framework.create_local_executor(NUM_CLIENTS))
tff.federated_computation(lambda: 'Hello, World!')()
#@test {"output": "ignore"}
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
emnist_train.output_types, emnist_train.output_shapes
example_dataset = emnist_train.create_tf_dataset_for_client(
emnist_train.client_ids[0])
example_element = iter(example_dataset).next()
example_element['label'].numpy()
NUM_EPOCHS = 10
BATCH_SIZE = 20
def recreate_tutorial(cuda_vislble_devices='1'):
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_vislble_devices
tff.federated_computation(lambda: 'Hello, World!')()
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
print(f'Number of clients: {len(emnist_train.client_ids)}')
print(f'Element type structure: {emnist_train.element_type_structure}')
example_dataset = emnist_train.create_tf_dataset_for_client(
emnist_train.client_ids[0])
example_element = next(iter(example_dataset))
print('Example element label:')
print(example_element['label'].numpy())
plt.imshow(example_element['pixels'].numpy(), cmap='gray', aspect='equal')
plt.grid(False)
plt.savefig('tmp.png')
preprocessed_example_dataset = preprocess(example_dataset)
sample_batch = tf.nest.map_structure(
lambda x: x.numpy(), next(iter(preprocessed_example_dataset)))
print('sample batch:')
print(sample_batch)
sample_clients = emnist_train.client_ids[0:NUM_CLIENTS]
federated_train_data = make_federated_data(emnist_train, sample_clients)
print('Number of client datasets: {l}'.format(l=len(federated_train_data)))
print('First dataset: {d}'.format(d=federated_train_data[0]))
def model_fn():
# We _must_ create a new model here, and _not_ capture it from an external
# scope. TFF will call this within different graph contexts.
keras_model = create_keras_model()
return tff.learning.from_keras_model(
keras_model,
input_spec=preprocessed_example_dataset.element_spec,
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
iterative_process = tff.learning.build_federated_averaging_process(
model_fn,
client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02
),
server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=1.0))
print('Iterative process type signature:')
print(str(iterative_process.initialize.type_signature))
state = iterative_process.initialize()
state, metrics = iterative_process.next(state, federated_train_data)
print('round 1, metrics={}'.format(metrics))
for round_num in range(2, NUM_ROUNDS):
state, metrics = iterative_process.next(state, federated_train_data)
print('round {:2d}, metrics={}'.format(round_num, metrics))
logdir = "/tmp/logs/scalars/training/"
summary_writer = tf.summary.create_file_writer(logdir)
state = iterative_process.initialize()
with summary_writer.as_default():
for round_num in range(1, NUM_ROUNDS):
state, metrics = iterative_process.next(state,
federated_train_data)
for name, value in metrics._asdict().items():
tf.summary.scalar(name, value, step=round_num)
def test():
''' Check that everything is in place.'''
test = tff.federated_computation(lambda: 'Hello, World!')()
assert test == b'Hello, World!'
print(test)
import six
from six.moves import range
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow_federated as tff
warnings.simplefilter('ignore')
tf.compat.v1.enable_v2_behavior()
np.random.seed(0)
if six.PY3:
tff.framework.set_default_executor(tff.framework.create_local_executor())
tff.federated_computation(lambda: 'Hello')()
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
dataset = emnist_train.create_tf_dataset_for_client(emnist_train.client_ids[0])
element = iter(dataset).next()
element['label'].numpy()
NUM_CLIENTS = 10
NUM_EPOCHS = 10
BATCH_SIZE = 20
SHUFFLE_BUFFER = 500
def preprocess(dataset):
def build_triehh_process(possible_prefix_extensions: List[str],
num_sub_rounds: int,
max_num_heavy_hitters: int,
max_user_contribution: int,
default_terminator: str = '$'):
"""Builds the TFF computations for heavy hitters discovery with TrieHH.
TrieHH works by interactively keeping track of popular prefixes. In each
round, the server broadcasts the popular prefixes it has
discovered so far and the list of `possible_prefix_extensions` to a small
fraction of selected clients. The select clients sample
`max_user_contributions` words from their local datasets, and use them to vote
on character extensions to the broadcasted popular prefixes. Client votes are
accumulated across `num_sub_rounds` rounds, and then the top
`max_num_heavy_hitters` extensions are used to extend the already discovered
prefixes, and the extended prefixes are used in the next round. When an
already discovered prefix is extended by `default_terminator` it is added to
the list of discovered heavy hitters.
Args:
possible_prefix_extensions: A list containing all the possible extensions to
learned prefixes. Each extensions must be a single character strings.
num_sub_rounds: The total number of sub rounds to be executed before
decoding aggregated votes. Must be positive.
max_num_heavy_hitters: The maximum number of discoverable heavy hitters.
Must be positive.
max_user_contribution: The maximum number of examples a user can contribute.
Must be positive.
default_terminator: The end of sequence symbol.
Returns:
A `tff.utils.IterativeProcess`.
"""
@tff.tf_computation
def server_init_tf():
return ServerState(
discovered_heavy_hitters=tf.constant([], dtype=tf.string),
discovered_prefixes=tf.constant([''], dtype=tf.string),
possible_prefix_extensions=tf.constant(
possible_prefix_extensions, dtype=tf.string),
round_num=tf.constant(0, dtype=tf.int32),
accumulated_votes=tf.zeros(
dtype=tf.int32,
shape=[max_num_heavy_hitters,
len(possible_prefix_extensions)]))
# We cannot use server_init_tf.type_signature.result because the
# discovered_* fields need to have [None] shapes, since they will grow over
# time.
server_state_type = (
tff.to_type(
ServerState(
discovered_heavy_hitters=tff.TensorType(
dtype=tf.string, shape=[None]),
discovered_prefixes=tff.TensorType(dtype=tf.string, shape=[None]),
possible_prefix_extensions=tff.TensorType(
dtype=tf.string, shape=[len(possible_prefix_extensions)]),
round_num=tff.TensorType(dtype=tf.int32, shape=[]),
accumulated_votes=tff.TensorType(
dtype=tf.int32, shape=[None,
len(possible_prefix_extensions)]),
)))
sub_round_votes_type = tff.TensorType(
dtype=tf.int32,
shape=[max_num_heavy_hitters,
len(possible_prefix_extensions)])
@tff.tf_computation(server_state_type, sub_round_votes_type)
@tf.function
def server_update_fn(server_state, sub_round_votes):
server_state = server_update(
server_state,
sub_round_votes,
num_sub_rounds=tf.constant(num_sub_rounds),
max_num_heavy_hitters=tf.constant(max_num_heavy_hitters),
default_terminator=tf.constant(default_terminator, dtype=tf.string))
return server_state
tf_dataset_type = tff.SequenceType(tf.string)
discovered_prefixes_type = tff.TensorType(dtype=tf.string, shape=[None])
round_num_type = tff.TensorType(dtype=tf.int32, shape=[])
@tff.tf_computation(tf_dataset_type, discovered_prefixes_type, round_num_type)
@tf.function
def client_update_fn(tf_dataset, discovered_prefixes, round_num):
result = client_update(tf_dataset, discovered_prefixes,
tf.constant(possible_prefix_extensions), round_num,
num_sub_rounds, max_num_heavy_hitters,
max_user_contribution)
return result
federated_server_state_type = tff.FederatedType(server_state_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(
tf_dataset_type, tff.CLIENTS, all_equal=False)
@tff.federated_computation(federated_server_state_type,
federated_dataset_type)
def run_one_round(server_state, federated_dataset):
"""Orchestration logic for one round of TrieHH computation.
Args:
server_state: A `ServerState`.
federated_dataset: A federated `tf.Dataset` with placement `tff.CLIENTS`.
Returns:
An updated `ServerState`
"""
discovered_prefixes = tff.federated_broadcast(
server_state.discovered_prefixes)
round_num = tff.federated_broadcast(server_state.round_num)
client_outputs = tff.federated_map(
client_update_fn,
tff.federated_zip([federated_dataset, discovered_prefixes, round_num]))
accumulated_votes = tff.federated_sum(client_outputs.client_votes)
server_state = tff.federated_map(server_update_fn,
(server_state, accumulated_votes))
server_output = tff.federated_value([], tff.SERVER)
return server_state, server_output
return tff.utils.IterativeProcess(
initialize_fn=tff.federated_computation(
lambda: tff.federated_value(server_init_tf(), tff.SERVER)),
next_fn=run_one_round)
def build_federated_averaging_process(
model_fn,
server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=1.0),
client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.1)):
"""Builds the TFF computations for optimization using federated averaging.
Args:
model_fn: A no-arg function that returns a `tff.learning.TrainableModel`.
server_optimizer_fn: A no-arg function that returns a
`tf.keras.optimizers.Optimizer` for server update.
client_optimizer_fn: A no-arg function that returns a
`tf.keras.optimizers.Optimizer` for client update.
Returns:
A `tff.utils.IterativeProcess`.
"""
dummy_model = model_fn(
) # TODO(b/144510813): try remove dependency on dummy model
@tff.tf_computation
def server_init_tf():
model = model_fn()
server_optimizer = server_optimizer_fn()
_initialize_optimizer_vars(model, server_optimizer)
return ServerState(model=model.weights,
optimizer_state=server_optimizer.variables())
server_state_type = server_init_tf.type_signature.result
model_weights_type = server_state_type.model
@tff.tf_computation(server_state_type, model_weights_type.trainable)
def server_update_fn(server_state, model_delta):
model = model_fn()
server_optimizer = server_optimizer_fn()
_initialize_optimizer_vars(model, server_optimizer)
return server_update(model, server_optimizer, server_state,
model_delta)
tf_dataset_type = tff.SequenceType(dummy_model.input_spec)
@tff.tf_computation(tf_dataset_type, model_weights_type)
def client_update_fn(tf_dataset, initial_model_weights):
model = model_fn()
client_optimizer = client_optimizer_fn()
return client_update(model, tf_dataset, initial_model_weights,
client_optimizer)
federated_server_state_type = tff.FederatedType(server_state_type,
tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
@tff.federated_computation(federated_server_state_type,
federated_dataset_type)
def run_one_round(server_state, federated_dataset):
"""Orchestration logic for one round of computation.
Args:
server_state: A `ServerState`.
federated_dataset: A federated `tf.Dataset` with placement `tff.CLIENTS`.
Returns:
A tuple of updated `ServerState` and the result of
`tff.learning.Model.federated_output_computation`.
"""
client_model = tff.federated_broadcast(server_state.model)
client_outputs = tff.federated_map(client_update_fn,
(federated_dataset, client_model))
weight_denom = client_outputs.client_weight
round_model_delta = tff.federated_mean(client_outputs.weights_delta,
weight=weight_denom)
server_state = tff.federated_map(server_update_fn,
(server_state, round_model_delta))
aggregated_outputs = dummy_model.federated_output_computation(
client_outputs.model_output)
aggregated_outputs = tff.federated_zip(aggregated_outputs)
return server_state, aggregated_outputs
return tff.utils.IterativeProcess(initialize_fn=tff.federated_computation(
lambda: tff.federated_value(server_init_tf(), tff.SERVER)),
next_fn=run_one_round)
if six.PY3:
print('Python3')
tff.framework.set_default_executor(tff.framework.create_local_executor())
def print_line():
print('------------------------------------------------------------')
print_line()
print('TensorFlow Version: ', tf.__version__)
print_line()
print('TensorFlow Federated Version: ', tff.__version__)
print_line()
print(
tff.federated_computation(lambda: 'TensorFlow Federated is working -_-')())
# Load data from file using Pandas
IoT_features = pd.read_csv('/home/user/csv_files/features_2016_10Min.csv',
sep=',')
print_line()
############################################################
# Normalization
############################################################
print('Normalization ...')
# Isolate labels from features
IoT_features_labels = IoT_features.pop('device_co')
# Normalize
ary = IoT_features.values
min_max_scaler = preprocessing.MinMaxScaler()
def build_triehh_process(
possible_prefix_extensions: List[str],
num_sub_rounds: int,
max_num_prefixes: int,
threshold: int,
max_user_contribution: int,
default_terminator: str = triehh_tf.DEFAULT_TERMINATOR):
"""Builds the TFF computations for heavy hitters discovery with TrieHH.
TrieHH works by interactively keeping track of popular prefixes. In each
round, the server broadcasts the popular prefixes it has
discovered so far and the list of `possible_prefix_extensions` to a small
fraction of selected clients. The select clients sample
`max_user_contributions` words from their local datasets, and use them to vote
on character extensions to the broadcasted popular prefixes. Client votes are
accumulated across `num_sub_rounds` rounds, and then the top
`max_num_prefixes` extensions get at least 'threshold' votes are used to
extend the already discovered
prefixes, and the extended prefixes are used in the next round. When an
already discovered prefix is extended by `default_terminator` it is added to
the list of discovered heavy hitters.
Args:
possible_prefix_extensions: A list containing all the possible extensions to
learned prefixes. Each extensions must be a single character strings. This
list should not contain the default_terminator.
num_sub_rounds: The total number of sub rounds to be executed before
decoding aggregated votes. Must be positive.
max_num_prefixes: The maximum number of prefixes we can keep in the trie.
Must be positive.
threshold: The threshold for heavy hitters and discovered prefixes. Only
those get at least `threshold` votes are discovered. Must be positive.
max_user_contribution: The maximum number of examples a user can contribute.
Must be positive.
default_terminator: The end of sequence symbol.
Returns:
A `tff.templates.IterativeProcess`.
Raises:
ValueError: If possible_prefix_extensions contains default_terminator.
"""
if default_terminator in possible_prefix_extensions:
raise ValueError(
'default_terminator should not appear in possible_prefix_extensions'
)
# Append `default_terminator` to `possible_prefix_extensions` to make sure it
# is the last item in the list.
possible_prefix_extensions.append(default_terminator)
@tff.tf_computation
def server_init_tf():
return ServerState(
discovered_heavy_hitters=tf.constant([], dtype=tf.string),
heavy_hitters_frequencies=tf.constant([], dtype=tf.float64),
discovered_prefixes=tf.constant([''], dtype=tf.string),
round_num=tf.constant(0, dtype=tf.int32),
accumulated_votes=tf.zeros(
dtype=tf.int32,
shape=[max_num_prefixes,
len(possible_prefix_extensions)]),
accumulated_weights=tf.constant(0, dtype=tf.int32))
# We cannot use server_init_tf.type_signature.result because the
# discovered_* fields need to have [None] shapes, since they will grow over
# time.
server_state_type = (tff.to_type(
ServerState(
discovered_heavy_hitters=tff.TensorType(dtype=tf.string,
shape=[None]),
heavy_hitters_frequencies=tff.TensorType(dtype=tf.float64,
shape=[None]),
discovered_prefixes=tff.TensorType(dtype=tf.string, shape=[None]),
round_num=tff.TensorType(dtype=tf.int32, shape=[]),
accumulated_votes=tff.TensorType(
dtype=tf.int32, shape=[None,
len(possible_prefix_extensions)]),
accumulated_weights=tff.TensorType(dtype=tf.int32, shape=[]),
)))
sub_round_votes_type = tff.TensorType(
dtype=tf.int32,
shape=[max_num_prefixes,
len(possible_prefix_extensions)])
sub_round_weight_type = tff.TensorType(dtype=tf.int32, shape=[])
@tff.tf_computation(server_state_type, sub_round_votes_type,
sub_round_weight_type)
def server_update_fn(server_state, sub_round_votes, sub_round_weight):
return server_update(server_state,
tf.constant(possible_prefix_extensions),
sub_round_votes,
sub_round_weight,
num_sub_rounds=tf.constant(num_sub_rounds),
max_num_prefixes=tf.constant(max_num_prefixes),
threshold=tf.constant(threshold))
tf_dataset_type = tff.SequenceType(tf.string)
discovered_prefixes_type = tff.TensorType(dtype=tf.string, shape=[None])
round_num_type = tff.TensorType(dtype=tf.int32, shape=[])
@tff.tf_computation(tf_dataset_type, discovered_prefixes_type,
round_num_type)
def client_update_fn(tf_dataset, discovered_prefixes, round_num):
return client_update(tf_dataset, discovered_prefixes,
tf.constant(possible_prefix_extensions),
round_num, num_sub_rounds, max_num_prefixes,
max_user_contribution)
federated_server_state_type = tff.FederatedType(server_state_type,
tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type,
tff.CLIENTS,
all_equal=False)
@tff.federated_computation(federated_server_state_type,
federated_dataset_type)
def run_one_round(server_state, federated_dataset):
"""Orchestration logic for one round of TrieHH computation.
Args:
server_state: A `ServerState`.
federated_dataset: A federated `tf.Dataset` with placement `tff.CLIENTS`.
Returns:
An updated `ServerState`
"""
discovered_prefixes = tff.federated_broadcast(
server_state.discovered_prefixes)
round_num = tff.federated_broadcast(server_state.round_num)
client_outputs = tff.federated_map(
client_update_fn,
(federated_dataset, discovered_prefixes, round_num))
accumulated_votes = tff.federated_sum(client_outputs.client_votes)
accumulated_weights = tff.federated_sum(client_outputs.client_weight)
server_state = tff.federated_map(
server_update_fn,
(server_state, accumulated_votes, accumulated_weights))
server_output = tff.federated_value([], tff.SERVER)
return server_state, server_output
return tff.templates.IterativeProcess(
initialize_fn=tff.federated_computation(
lambda: tff.federated_eval(server_init_tf, tff.SERVER)),
next_fn=run_one_round)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
# tutorial: https://www.tensorflow.org/federated/tutorials/federated_learning_for_image_classification
import warnings
import collections
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
# check that environment is correctly set up
warnings.simplefilter('ignore')
tf.compat.v1.enable_v2_behavior()
np.random.seed(0)
tff.framework.set_default_executor(tff.framework.create_local_executor())
assert tff.federated_computation(lambda: 'Hello, World!')() == b'Hello, World!'
# variables are tff.simulation.ClientData objects
(emnist_train, emnist_test) = tff.simulation.datasets.emnist.load_data()
assert len(emnist_train.client_ids) == 3383
assert emnist_train.element_type_structure == OrderedDict([
('label', TensorSpec(shape=(), dtype=tf.int32, name=None)),
('pixels', TensorSpec(shape=(28, 28), dtype=tf.float32, name=None))
])
# creates a new tf.data.Dataset containing the client[0] training examples
example_dataset = emnist_train.create_tf_dataset_for_client(
emnist_train.client_ids[0])
# test for specific datum