def get_recordio_protobuf_dmatrix(path,
is_pipe=False,
subsample_ratio_on_read=None):
"""Get Data Matrix from recordio-protobuf data.
:param path: Path where recordio-protobuf formatted training data resides, either directory, file, or SageMaker pipe
:param is_pipe: Boolean to indicate if data is being read in pipe mode
:param subsample_ratio_on_read: None or a value in (0, 1) to indicate how much of the dataset should
be read into memory.
:return: xgb.DMatrix or None
"""
try:
if is_pipe:
dataset = [mlio.SageMakerPipe(path)]
reader = mlio.RecordIOProtobufReader(
dataset=dataset,
batch_size=BATCH_SIZE,
subsample_ratio=subsample_ratio_on_read)
else:
dataset = mlio.list_files(path)
reader = mlio.RecordIOProtobufReader(
dataset=dataset,
batch_size=BATCH_SIZE,
subsample_ratio=subsample_ratio_on_read)
exm = reader.peek_example()
if exm is None:
return None
# Recordio-protobuf tensor may be dense (use numpy) or sparse (use scipy)
if isinstance(exm['values'], mlio.DenseTensor):
to_matrix = as_numpy
vstack = np.vstack
else:
to_matrix = to_coo_matrix
vstack = scipy_vstack
all_values = []
all_labels = []
for example in reader:
values = to_matrix(example['values'])
all_values.append(values)
labels = as_numpy(example['label_values']).squeeze()
all_labels.append(labels)
all_values = vstack(all_values)
all_labels = np.concatenate(all_labels)
return xgb.DMatrix(all_values, label=all_labels)
except Exception as e:
raise exc.UserError(
"Failed to load recordio-protobuf data with exception:\n{}".format(
e))
def test_recordio_protobuf_reader_params():
filename = os.path.join(resources_dir, 'test.pbr')
dataset = [mlio.File(filename)]
rdr_prm = mlio.DataReaderParams(dataset=dataset, batch_size=1)
reader = mlio.RecordIOProtobufReader(rdr_prm)
example = reader.read_example()
record = [as_numpy(feature) for feature in example]
assert record[0].squeeze() == np.array(1)
assert np.all(record[1].squeeze() == np.array([0, 0, 0]))
# Parameters should be reusable
reader2 = mlio.RecordIOProtobufReader(rdr_prm)
assert reader2.peek_example()
def recordio_protobuf_to_dmatrix(string_like): # type: (bytes) -> xgb.DMatrix
"""Convert a RecordIO-Protobuf byte representation to a DMatrix object.
Args:
string_like (bytes): RecordIO-Protobuf bytes.
Returns:
(xgb.DMatrix): XGBoost DataMatrix
"""
buf = bytes(string_like)
dataset = [mlio.InMemoryStore(buf)]
reader_params = mlio.DataReaderParams(dataset=dataset, batch_size=100)
reader = mlio.RecordIOProtobufReader(reader_params)
is_dense_tensor = type(reader.peek_example()['values']) is mlio.DenseTensor
examples = []
for example in reader:
# Ignore labels if present
values = as_numpy(
example['values']) if is_dense_tensor else to_coo_matrix(
example['values'])
examples.append(values)
data = np.vstack(examples) if is_dense_tensor else scipy_vstack(
examples).tocsr()
dmatrix = xgb.DMatrix(data)
return dmatrix
def recordio_protobuf_to_dmatrix(string_like): # type: (bytes) -> xgb.DMatrix
"""Convert a RecordIO-Protobuf byte representation to a DMatrix object.
Args:
string_like (bytes): RecordIO-Protobuf bytes.
Returns:
(xgb.DMatrix): XGBoost DataMatrix
"""
buf = bytes(string_like)
dataset = [mlio.InMemoryStore(buf)]
reader = mlio.RecordIOProtobufReader(dataset=dataset, batch_size=100)
if type(reader.peek_example()['values']) is mlio.core.DenseTensor:
to_matrix = as_numpy
vstack = np.vstack
else:
to_matrix = to_coo_matrix
vstack = scipy_vstack
examples = []
for example in reader:
tmp = to_matrix(example['values']) # Ignore labels if present
examples.append(tmp)
data = vstack(examples)
dmatrix = xgb.DMatrix(data)
return dmatrix
def pipe_iterator(self, fifo_id=0):
fifo_id = self.increment()
print(f"opening pipe iterator {self.path}:{fifo_id}")
pipe = mlio.SageMakerPipe(self.path, fifo_id=fifo_id)
reader_params = mlio.DataReaderParams(dataset=[pipe],
**self.reader_params)
reader = mlio.RecordIOProtobufReader(reader_params)
return reader
def get_recordio_protobuf_dmatrix(path, is_pipe=False):
"""Get Data Matrix from recordio-protobuf data.
:param path: Path where recordio-protobuf formatted training data resides, either directory, file, or SageMaker pipe
:param is_pipe: Boolean to indicate if data is being read in pipe mode
:return: xgb.DMatrix or None
"""
try:
if is_pipe:
dataset = [mlio.SageMakerPipe(path)]
reader = mlio.RecordIOProtobufReader(dataset=dataset,
batch_size=BATCH_SIZE)
else:
dataset = mlio.list_files(path)
reader = mlio.RecordIOProtobufReader(dataset=dataset,
batch_size=BATCH_SIZE)
if reader.peek_example() is not None:
# recordio-protobuf tensor may be dense (use numpy) or sparse (use scipy)
if type(reader.peek_example()['values']) is mlio.core.DenseTensor:
to_matrix = as_numpy
vstack = np.vstack
else:
to_matrix = to_coo_matrix
vstack = scipy_vstack
all_features = []
all_labels = []
for example in reader:
features = to_matrix(example['values'])
all_features.append(features)
labels = as_numpy(example['label_values'])
all_labels.append(labels)
all_features = vstack(all_features)
all_labels = np.concatenate(all_labels, axis=None)
dmatrix = xgb.DMatrix(all_features, label=all_labels)
return dmatrix
else:
return None
except Exception as e:
raise exc.UserError(
"Failed to load recordio-protobuf data with exception:\n{}".format(
e))