def test_create_network_hidden_units_number_properly_initialized(layer_number): model_data = { 'type': 'FC-DNN', 'hidden-units': [ { 'id': 'output', 'type': 'Dense', 'nodes': 10 }, ], 'loss': 'SoftmaxCrossEntropyLoss' } layer = { 'id': 'layer', 'type': 'Dense', 'nodes': 128, 'activation': 'relu' } for index in range(layer_number): new_layer = copy(layer) new_layer['id'] += str(index) model_data['hidden-units'].insert(0, new_layer) network = create_network(model_data) assert len(network) == layer_number + 1
def test_create_network_flatten_layer(): model_data = { 'type': 'FC-DNN', 'hidden-units': [ { 'id': 'layer1', 'type': 'Dense', 'nodes': 128, 'activation': 'relu' }, { 'id': 'embedding', 'type': 'Flatten', }, { 'id': 'output', 'type': 'Dense', 'nodes': 10 }, ], 'loss': 'SoftmaxCrossEntropyLoss' } network = create_network(model_data) assert type(network[1]) == nn.Flatten
def test_create_network_layer_normalization(): model_data = { 'type': 'FC-DNN', 'hidden-units': [ { 'id': 'layer1', 'type': 'Dense', 'nodes': 128, 'activation': 'relu' }, { 'id': 'batch1', 'type': 'LayerNorm' }, { 'id': 'output', 'type': 'Dense', 'nodes': 10 }, ], 'loss': 'SoftmaxCrossEntropyLoss' } network = create_network(model_data) assert type(network[1]) == nn.LayerNorm
def test_create_network_dropout_layer(): model_data = { 'type': 'FC-DNN', 'hidden-units': [ { 'id': 'layer1', 'type': 'Dense', 'nodes': 128, 'activation': 'relu' }, { 'id': 'dropout1', 'type': 'Dropout', 'rate': 0.5, }, { 'id': 'output', 'type': 'Dense', 'nodes': 10 }, ], 'loss': 'SoftmaxCrossEntropyLoss' } network = create_network(model_data) assert type(network[1]) == nn.Dropout
def test_create_network_fc_dnn(): model_data = { 'type': 'FC-DNN', 'hidden-units': [ { 'id': 'layer1', 'type': 'Dense', 'nodes': 128, 'activation': 'relu' }, { 'id': 'layer2', 'type': 'Dense', 'nodes': 64, 'activation': 'relu' }, { 'id': 'output', 'type': 'Dense', 'nodes': 10 }, ], 'loss': 'SoftmaxCrossEntropyLoss' } network = create_network(model_data) assert type(network) == nn.Sequential
def test_create_network_node_number_in_dense_layer(node_number): model_data = { 'type': 'FC-DNN', 'hidden-units': [ { 'id': 'layer1', 'type': 'Dense', 'nodes': node_number, 'activation': 'relu' }, { 'id': 'output', 'type': 'Dense', 'nodes': 10 }, ], 'loss': 'SoftmaxCrossEntropyLoss' } network = create_network(model_data) assert network[0]._units == node_number
def verify_iteration(msg_history_id, msg_id, nonce, block_header, redis_host='localhost', redis_port=6379): print('\nVERIFY ITERATION(msg_history_id = %s, msg_id = %s, nonce = %s)' % (msg_history_id, msg_id, nonce)) ctx = mx.cpu() conn = redis.Redis(host=redis_host, port=redis_port) conn.ping() json_maps = conn.mget(msg_id) iterations_data = [ json.loads(data) for data in json_maps if data is not None ] if len(iterations_data) == 0: print('VERIFICATION FAILED -- No message found in Redis.') return verifier_pb2.Response(code=verifier_pb2.Response.NOT_FOUND, description="Message not found in Redis.") it_data = iterations_data[0] worker_id = it_data['signature'] task_id = it_data['task_id'] batch_location = it_data['batch_hash'] model_hash = it_data['model_hash'] learning_rate = float(it_data['e_l']) error_code, reason = verify_block_commitment(conn, msg_id, worker_id, block_header) if error_code is not None: print(f'VERIFICATION FAILED -- {reason}') return verifier_pb2.Response(code=error_code, description=reason) model_template = 'models/' + task_id + '/' + worker_id + '/' + model_hash + '/model.params' print('Model: %s' % model_template) try: os.makedirs(TEMP_FOLDER, exist_ok=True) model_file_name = str(uuid.uuid4()) model_location = TEMP_FOLDER + model_file_name shutil.copyfile(os.path.join(BUCKET, model_template), model_location) except Exception as e: print('download_file exception: %s' % e) traceback.print_exc() raise try: with open( os.path.join(os.path.dirname(__file__), '../client-task-definition.yaml'), 'r') as request_file: request_data = yaml.load(request_file, yaml.UnsafeLoader) ver_net = create_network(request_data['ml']['model']) ver_net.load_parameters(model_location) except Exception as e: print('ver_net exception: %s' % e) raise print('Epoch: %d' % (it_data['epoch'])) batch_features_template = 'batches/' + batch_location + '/features' batch_labels_template = 'batches/' + batch_location + '/labels' batch_features_filename = str(uuid.uuid4()) batch_features_location = TEMP_FOLDER + batch_features_filename shutil.copyfile(os.path.join(BUCKET, batch_features_template), batch_features_location) batch_labels_filename = str(uuid.uuid4()) batch_labels_location = TEMP_FOLDER + batch_labels_filename shutil.copyfile(os.path.join(BUCKET, batch_labels_template), batch_labels_location) # Trainer is for updating parameters with gradient. We use SGD as optimizer. trainer = gluon.Trainer(ver_net.collect_params(), 'sgd', { 'learning_rate': learning_rate, 'momentum': 0.0 }) # Add metrics: accuracy and cross-entropy loss accuracy = mx.metric.Accuracy() ce_loss = mx.metric.CrossEntropy() comp_metric = mx.metric.CompositeEvalMetric([accuracy, ce_loss]) loss = gluon.loss.SoftmaxCrossEntropyLoss() try: data = mx.nd.load(batch_features_location) label = mx.nd.load(batch_labels_location) except: print('VERIFICATION FAILED. Could not load data batch.') return verifier_pb2.Response(code=verifier_pb2.Response.NOT_FOUND, description="Could not load data batch.") data = data[0].as_in_context(ctx) label = label[0].as_in_context(ctx) comp_metric.reset() peer_msg_map = it_data['peer_msg_ids'] other_workers_data = get_other_workers_local_data(conn, peer_msg_map) # perform gradients modifications here grads = [ g.grad(ctx) for g in ver_net.collect_params().values() if g._grad is not None ] gradients_blueprint = [g.shape for g in grads] gradients_sizes = [g.size for g in grads] gradients_cumulative = np.insert(np.cumsum(gradients_sizes), 0, 0)[:-1] zero_setter = np.vectorize(lambda int_type: int_type & (~(1 << 31)), otypes=[np.uint32]) for worker_data in other_workers_data: local_map = worker_data['local_deltas'] deltas = pai.pouw.message_map.decode_message_map( ctx, local_map, gradients_blueprint, gradients_cumulative, worker_data['tau'], zero_setter) trainer.allreduce_grads() # do back propagation with autograd.record(): output = ver_net(data) L = loss(output, label) L.backward() for idx, grad in enumerate(grads): grad *= 0 grad += deltas[idx].as_in_context(ctx) # take a gradient step with batch_size equal to data.shape[0] trainer.update(data.shape[0]) trainer.allreduce_grads() with autograd.record(): output = ver_net(data) L = loss(output, label) # do back propagation L.backward() initial_local_gradients = overdrive.clone_gradients(grads) weight_indices = it_data['local_deltas'] deltas = message_map.decode_message_map(ctx, weight_indices, gradients_blueprint, gradients_cumulative, it_data['tau'], zero_setter) for idx, grad in enumerate(grads): grad *= 0 grad += deltas[idx].as_in_context(ctx) # take a gradient step with batch_size equal to data.shape[0] trainer.update(data.shape[0]) # calculate overdrive tau = it_data['tau'] overdrive_index = overdrive.calculate_overdrive(initial_local_gradients, deltas, tau) # re-evaluate network output output = ver_net(data) # update metrics comp_metric.update([label], [output.softmax()]) names, metrics = comp_metric.get() mini_batch_actual = get_batch_hash(data, label) mini_batch_provided = it_data['batch_hash'] model_hash_actual = file_sha256_hexdigest(model_location) batches_ok = mini_batch_actual == mini_batch_provided print('Mini-batch hashes match: %s -> actual : %s | provided: %s' % ('YES' if batches_ok else 'NO', mini_batch_actual, mini_batch_provided)) if not batches_ok: print('VERIFICATION FAILED') return verifier_pb2.Response(code=verifier_pb2.Response.NOT_FOUND, description="Batches don't match.") models_ok = model_hash_actual == model_hash print('Model hashes match: %s -> actual : %s | provided: %s' % ('YES' if models_ok else 'NO', model_hash_actual, model_hash)) if not models_ok: print('VERIFICATION FAILED') return verifier_pb2.Response(code=verifier_pb2.Response.NOT_FOUND, description="Models don't match.") accuracies_ok = metrics[0] == it_data['acc_tr'] print('Accuracies match: %s -> actual : %s | provided: %s' % ('YES' if accuracies_ok else 'NO', metrics[0], it_data['acc_tr'])) if not accuracies_ok: print('VERIFICATION FAILED') return verifier_pb2.Response( code=verifier_pb2.Response.INVALID, description="Invalid accuracy.\nGot {}, expected {}".format( metrics[0], it_data['acc_tr'])) loss_ok = metrics[1] == it_data['j_tr'] print('Loss values match: %s -> actual : %s | provided: %s' % ('YES' if loss_ok else 'NO', metrics[1], it_data['j_tr'])) if not loss_ok: print('VERIFICATION FAILED') return verifier_pb2.Response( code=verifier_pb2.Response.INVALID, description="Invalid loss.\nGot {}, expected {}".format( metrics[1], it_data['j_tr'])) overdrive_index_ok = overdrive_index == it_data['overdrive'] print('Overdrive index values match: %s -> actual : %s | provided: %s' % ('YES' if overdrive_index_ok else 'NO', overdrive_index, it_data['overdrive'])) if not overdrive_index_ok: print('VERIFICATION FAILED') return verifier_pb2.Response(code=verifier_pb2.Response.INVALID, description="Invalid overdrive index.") # verify nonce os.makedirs(TEMP_FOLDER, exist_ok=True) ver_net.save_parameters(TEMP_FOLDER + 'end_it_model.params') end_it_model_hash = file_sha256_digest(TEMP_FOLDER + 'end_it_model.params') actual_nonce = Miner.calculate_nonce(end_it_model_hash, weight_indices) actual_nonce_int = swap32(int(binascii.hexlify(actual_nonce), 16)) nonce_ok = actual_nonce_int == nonce print('Nonces match: %s -> actual : %s | provided: %s' % ('YES' if nonce_ok else 'NO', actual_nonce_int, nonce)) if not nonce_ok: print('VERIFICATION FAILED') return verifier_pb2.Response(code=verifier_pb2.Response.INVALID, description="Nonces don't match.") # clean up resources if os.path.exists(model_location): os.remove(model_location) if os.path.exists(batch_features_location): os.remove(batch_features_location) if os.path.exists(batch_labels_location): os.remove(batch_labels_location) if os.path.isfile(TEMP_FOLDER + 'end_it_model.params'): os.remove(TEMP_FOLDER + 'end_it_model.params') print('VERIFICATION SUCCESSFUL') return verifier_pb2.Response(code=verifier_pb2.Response.OK, description="Verification successful.")