def get_merged(bucket_name, prefix, file_postfix): # print('get merged weight {} in bucket {}'.format(w_prefix + file_postfix, bucket_name)) merged_value = get_object_or_wait(bucket_name, prefix + file_postfix, 0.01).read() merged_value_np = pickle.loads(merged_value) # merged_value_np = np.frombuffer(merged_value, dtype=dtype).reshape(dshape) return merged_value_np
def get_merged_w_b(bucket_name, file_postfix, dtype, w_shape, b_shape="", w_prefix="w_", b_prefix="b_"): print('get merged weight {} in bucket {}'.format(w_prefix + file_postfix, bucket_name)) w_data = get_object_or_wait(bucket_name, w_prefix + file_postfix, 0.1).read() w_grad = np.frombuffer(w_data, dtype=dtype).reshape(w_shape) print('get merged bias {} in bucket {}'.format(b_prefix + file_postfix, bucket_name)) b_data = get_object_or_wait(bucket_name, b_prefix + file_postfix, 0.1).read() b_grad = np.frombuffer(b_data, dtype=dtype).reshape(b_shape) return w_grad, b_grad
def reduce_batch(vector, tmp_bucket, merged_bucket, num_workers, worker_index, postfix): # vector is supposed to be a 1-d numpy array vec_shape = vector.shape vec_dtype = vector.dtype merged_vec = np.zeros(vec_shape, dtype=vec_dtype) postfix_splits = postfix.split("_") curr_epoch = int(postfix_splits[0]) curr_batch = int(postfix_splits[1]) # put object to s3, format of key: workerID_epoch_batch key = "{}_{}".format(worker_index, postfix) put_object(tmp_bucket, key, vector.tobytes()) # the first worker read and aggregate the corresponding chunk if worker_index == 0: num_files = 0 while num_files < num_workers: objects = list_bucket_objects(tmp_bucket) if objects is not None: delete_list = [] for obj in objects: file_key = urllib.parse.unquote_plus(obj["Key"], encoding='utf-8') key_splits = file_key.split("_") key_epoch = key_splits[1] key_batch = key_splits[2] if key_epoch == str(curr_epoch) and key_batch == str( curr_batch): data = get_object(tmp_bucket, file_key).read() bytes_data = np.frombuffer(data, dtype=vec_dtype) tmp_vec = bytes_data.reshape(vec_shape) merged_vec += tmp_vec num_files += 1 delete_list.append(file_key) delete_objects(tmp_bucket, delete_list) # write the merged data back to s3 merged_file_name = 'merged_' + postfix put_object(merged_bucket, merged_file_name, merged_vec.tobytes()) delete_expired_merged_batch(merged_bucket, curr_epoch, curr_batch) else: merged_file_name = 'merged_' + postfix merged_data = get_object_or_wait(merged_bucket, merged_file_name, 0.1).read() merged_vec = np.frombuffer(merged_data, dtype=vec_dtype).reshape(vec_shape) return merged_vec
def handler(event, context): start_time = time.time() bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8') print('bucket = {}'.format(bucket)) print('key = {}'.format(key)) key_splits = key.split("_") worker_index = int(key_splits[0]) num_worker = int(key_splits[1]) sync_meta = SyncMeta(worker_index, num_worker) print("synchronization meta {}".format(sync_meta.__str__())) # read file from s3 file = get_object(bucket, key).read().decode('utf-8').split("\n") print("read data cost {} s".format(time.time() - start_time)) parse_start = time.time() dataset = DenseDatasetWithLines(file, num_features) print("parse data cost {} s".format(time.time() - parse_start)) preprocess_start = time.time() # Creating data indices for training and validation splits: dataset_size = len(dataset) indices = list(range(dataset_size)) split = int(np.floor(validation_ratio * dataset_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) validation_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler) print("preprocess data cost {} s".format(time.time() - preprocess_start)) model = LogisticRegression(num_features, num_classes) # Loss and Optimizer # Softmax is internally computed. # Set parameters to be updated. criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) # Training the Model for epoch in range(num_epochs): for batch_index, (items, labels) in enumerate(train_loader): print("------worker {} epoch {} batch {}------".format( worker_index, epoch, batch_index)) batch_start = time.time() items = Variable(items.view(-1, num_features)) labels = Variable(labels) # Forward + Backward + Optimize optimizer.zero_grad() outputs = model(items) loss = criterion(outputs, labels) loss.backward() print("forward and backward cost {} s".format(time.time() - batch_start)) optimizer.step() w = model.linear.weight.data.numpy() b = model.linear.bias.data.numpy() #print("dtype of grad = {}".format(w_grad.dtype)) print("weight before update = {}".format(w[0][0:5])) print("bias before update = {}".format(b)) sync_start = time.time() put_object(model_bucket, w_prefix, w.tobytes()) put_object(model_bucket, b_prefix, b.tobytes()) w_data = get_object_or_wait(model_bucket, w_prefix, 0.1).read() w_new = np.frombuffer(w_data, dtype=w.dtype).reshape(w.shape) b_data = get_object_or_wait(model_bucket, b_prefix, 0.1).read() b_new = np.frombuffer(b_data, dtype=b.dtype).reshape(b.shape) model.linear.weight.data = torch.from_numpy(w_new) model.linear.bias.data = torch.from_numpy(b_new) print("weight after update = {}".format( model.linear.weight.data.numpy()[0][:5])) print("b_grad after update = {}".format( model.linear.bias.data.numpy())) print("synchronization cost {} s".format(time.time() - sync_start)) print("batch cost {} s".format(time.time() - batch_start)) if (batch_index + 1) % 10 == 0: print('Epoch: [%d/%d], Step: [%d/%d], Loss: %.4f' % (epoch + 1, num_epochs, batch_index + 1, len(train_indices) / batch_size, loss.data)) if worker_index == 0: clear_bucket(model_bucket) # Test the Model correct = 0 total = 0 for items, labels in validation_loader: items = Variable(items.view(-1, num_features)) # items = Variable(items) outputs = model(items) _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum() print('Accuracy of the model on the %d test samples: %d %%' % (len(val_indices), 100 * correct / total)) end_time = time.time() print("elapsed time = {} s".format(end_time - start_time))