def _test_dedupe_column_names(tmpdir,
input_column_names: List[str],
input_data: List[int],
expected_column_names: List[str],
expected_data: List[int],
dedupe_column_names: bool = True,
**kwargs) -> None:
header_str = ','.join(input_column_names)
data_str = ','.join(str(x) for x in input_data)
csv_file = tmpdir.join("test.csv")
csv_file.write(header_str + '\n' + data_str)
dataset = [mlio.File(str(csv_file))]
reader_params = mlio.DataReaderParams(dataset=dataset, batch_size=1)
csv_params = mlio.CsvParams(dedupe_column_names=dedupe_column_names,
**kwargs)
reader = mlio.CsvReader(reader_params, csv_params)
example = reader.read_example()
names = [desc.name for desc in example.schema.descriptors]
assert names == expected_column_names
record = [as_numpy(feature) for feature in example]
assert np.all(np.array(record).squeeze() == np.array(expected_data))
def recordio_protobuf_to_dmatrix(string_like): # type: (bytes) -> xgb.DMatrix
"""Convert a RecordIO-Protobuf byte representation to a DMatrix object.
Args:
string_like (bytes): RecordIO-Protobuf bytes.
Returns:
(xgb.DMatrix): XGBoost DataMatrix
"""
buf = bytes(string_like)
dataset = [mlio.InMemoryStore(buf)]
reader_params = mlio.DataReaderParams(dataset=dataset, batch_size=100)
reader = mlio.RecordIOProtobufReader(reader_params)
is_dense_tensor = type(reader.peek_example()['values']) is mlio.DenseTensor
examples = []
for example in reader:
# Ignore labels if present
values = as_numpy(
example['values']) if is_dense_tensor else to_coo_matrix(
example['values'])
examples.append(values)
data = np.vstack(examples) if is_dense_tensor else scipy_vstack(
examples).tocsr()
dmatrix = xgb.DMatrix(data)
return dmatrix
def _get_reader(source, batch_size):
"""Returns 'CsvReader' for the given source
Parameters
----------
source: str or bytes
Name of the SageMaker Channel, File, or directory from which the data is being read or
the Python buffer object from which the data is being read.
batch_size : int
The batch size in rows to read from the source.
Returns
-------
mlio.CsvReader
CsvReader configured with a SageMaker Pipe, File or InMemory buffer
"""
data_reader_params = mlio.DataReaderParams(dataset=_get_data(source),
batch_size=batch_size,
warn_bad_instances=False)
csv_params = mlio.CsvParams(default_data_type=mlio.DataType.STRING,
header_row_index=None,
allow_quoted_new_lines=True)
return mlio.CsvReader(data_reader_params=data_reader_params,
csv_params=csv_params)
def pipe_iterator(self, fifo_id=0):
fifo_id = self.increment()
print(f"opening pipe iterator {self.path}:{fifo_id}")
pipe = mlio.SageMakerPipe(self.path, fifo_id=fifo_id)
reader_params = mlio.DataReaderParams(dataset=[pipe],
**self.reader_params)
reader = mlio.RecordIOProtobufReader(reader_params)
return reader
def test_csv_params():
filename = os.path.join(resources_dir, 'test.csv')
dataset = [mlio.File(filename)]
rdr_prm = mlio.DataReaderParams(dataset=dataset, batch_size=1)
csv_prm = mlio.CsvParams(header_row_index=None)
reader = mlio.CsvReader(rdr_prm, csv_prm)
example = reader.read_example()
record = [as_numpy(feature) for feature in example]
assert np.all(np.array(record).squeeze() == np.array([1, 0, 0, 0]))
reader2 = mlio.CsvReader(rdr_prm, csv_prm)
assert reader2.peek_example()
async def __predict(self):
if self.params.ml_lib == 'snap':
from pai4sk import BoostingMachine as Booster
else:
from sklearn.tree import DecisionTreeRegressor
chunk_size = self.params.chunk_size # getattr(self.params, "chunk_size")
dataset = mlio.list_files(getattr(self.params, "dataset_test_path"),
pattern='*.csv')
logging.debug('mlio dataset={}'.format(dataset))
reader_params = mlio.DataReaderParams(
dataset=dataset,
batch_size=chunk_size,
num_prefetched_batches=self.params.num_prefetched_chunks)
reader = mlio.CsvReader(reader_params)
logging.debug('mlio reader={}'.format(reader))
logging.debug('starting inference')
score_norm = 0.0
score = 0.0
# preample
chunkim1 = reader.read_example()
if chunkim1 != None:
X_im1, y_im1 = await self.__preprocess_chunk(chunkim1)
chunki = reader.read_example()
i = 1
logging.debug('chunk{}={}'.format(0, chunkim1))
logging.debug('chunk{}={}'.format(i, chunki))
while chunki != None:
logging.debug('chunk{}={}'.format(i, chunki))
task_predict = asyncio.create_task(
self.__predict_chunk(X_im1, y_im1))
task_preprocess = asyncio.create_task(
self.__preprocess_chunk(chunki))
X_i, y_i = await task_preprocess
s, n = await task_predict
score += s
score_norm += n
X_im1 = X_i
y_im1 = y_i
chunkim1 = chunki
chunki = reader.read_example()
i += 1
# postample
if chunkim1 != None:
logging.debug('y{}m1={}'.format(i, y_im1))
s, n = await self.__predict_chunk(X_im1, y_im1)
score += s
score_norm += n
score /= score_norm
return score
def test_recordio_protobuf_reader_params():
filename = os.path.join(resources_dir, 'test.pbr')
dataset = [mlio.File(filename)]
rdr_prm = mlio.DataReaderParams(dataset=dataset, batch_size=1)
reader = mlio.RecordIOProtobufReader(rdr_prm)
example = reader.read_example()
record = [as_numpy(feature) for feature in example]
assert record[0].squeeze() == np.array(1)
assert np.all(record[1].squeeze() == np.array([0, 0, 0]))
# Parameters should be reusable
reader2 = mlio.RecordIOProtobufReader(rdr_prm)
assert reader2.peek_example()
def test_image_reader_jpeg_no_resize():
filename = os.path.join(resources_dir, 'test_image_0.jpg')
dataset = [mlio.File(filename)]
rdr_prm = mlio.DataReaderParams(dataset=dataset, batch_size=1)
img_prm = mlio.ImageReaderParams(img_frame=mlio.ImageFrame.NONE,
image_dimensions=[3, 50, 50],
to_rgb=1)
reader = mlio.ImageReader(rdr_prm, img_prm)
example = reader.read_example()
tensor = example['value']
assert tensor.shape == (1, 50, 50, 3)
assert tensor.strides == (7500, 150, 3, 1)
async def __train_old(self):
chunk_size = self.params.chunk_size # getattr(self.params, "chunk_size")
dataset = mlio.list_files(getattr(self.params, "dataset_path"),
pattern='*.csv')
logging.debug('mlio dataset={}'.format(dataset))
preproc_fn = self.params.preproc_fn
reader_params = mlio.DataReaderParams(
dataset=dataset,
batch_size=chunk_size,
num_prefetched_batches=self.params.num_prefetched_chunks)
reader = mlio.CsvReader(reader_params)
logging.debug('mlio reader={}'.format(reader))
num_epochs = self.params.num_epochs # Number of times to read the full dataset.
# use eta parameteres
eta = 0.01
if self.params.ml_lib == 'snap':
eta = 0.1
from pai4sk import BoostingMachine as Booster
else:
from sklearn.tree import DecisionTreeRegressor
logging.debug('starting training')
models = []
# RecordIOProtobufReader is simply an iterator over mini-batches of data.
for chunk_idx, chunk in enumerate(reader):
rand_state = self.params.rand_state
# Alternatively, transform the mini-batch into a NumPy array.
chunk_train_Xy = np.column_stack(
[as_numpy(feature) for feature in chunk])
chunk_train_X, chunk_train_y = preproc_fn(
chunk_train_Xy, self.params.label_col_idx)
#print(chunk_train_X)
if self.params.ml_lib == 'snap':
bl = Booster(**self.params.ml_opts_dict)
bl.fit(chunk_train_X, chunk_train_y)
models.append(bl)
else:
z_train = np.zeros(chunk_train_X.shape[0])
for epoch in range(num_epochs):
#logging.debug('chunk idx={} chunk={}'.format(chunk_idx, chunk))
target = chunk_train_y - z_train
bl = DecisionTreeRegressor(max_depth=3,
max_features='sqrt',
random_state=rand_state)
bl.fit(chunk_train_X, target)
u_train = bl.predict(chunk_train_X)
z_train = z_train + eta * u_train
models.append(bl)
return models
def get_recordio_protobuf_dmatrix(path, is_pipe=False):
"""Get Data Matrix from recordio-protobuf data.
:param path: Path where recordio-protobuf formatted training data resides, either directory, file, or SageMaker pipe
:param is_pipe: Boolean to indicate if data is being read in pipe mode
:return: xgb.DMatrix or None
"""
try:
if is_pipe:
pipes_path = path if isinstance(path, list) else [path]
dataset = [
mlio.SageMakerPipe(pipe_path) for pipe_path in pipes_path
]
else:
dataset = mlio.list_files(path)
reader_params = mlio.DataReaderParams(dataset=dataset,
batch_size=BATCH_SIZE)
reader = mlio.RecordIOProtobufReader(reader_params)
if reader.peek_example() is not None:
# recordio-protobuf tensor may be dense (use numpy) or sparse (use scipy)
is_dense_tensor = type(
reader.peek_example()['values']) is mlio.DenseTensor
all_features = []
all_labels = []
for example in reader:
features = as_numpy(
example['values']) if is_dense_tensor else to_coo_matrix(
example['values'])
all_features.append(features)
labels = as_numpy(example['label_values'])
all_labels.append(labels)
all_features = np.vstack(
all_features) if is_dense_tensor else scipy_vstack(
all_features).tocsr()
all_labels = np.concatenate(all_labels, axis=None)
dmatrix = xgb.DMatrix(all_features, label=all_labels)
return dmatrix
else:
return None
except Exception as e:
raise exc.UserError(
"Failed to load recordio-protobuf data with exception:\n{}".format(
e))
def test_image_reader_recordio():
filename = os.path.join(resources_dir, 'test_image_0.rec')
dataset = [mlio.File(filename)]
rdr_prm = mlio.DataReaderParams(dataset=dataset, batch_size=1)
img_prm = mlio.ImageReaderParams(img_frame=mlio.ImageFrame.RECORDIO,
resize=100,
image_dimensions=[3, 100, 100],
to_rgb=1)
reader = mlio.ImageReader(rdr_prm, img_prm)
example = reader.read_example()
tensor = example['value']
assert tensor.shape == (1, 100, 100, 3)
assert tensor.strides == (30000, 300, 3, 1)
def test_csv_nonutf_encoding_with_encoding_param():
filename = os.path.join(resources_dir, 'test_iso8859_5.csv')
dataset = [mlio.File(filename)]
rdr_prm = mlio.DataReaderParams(dataset=dataset,
batch_size=2)
csv_params = mlio.CsvParams(encoding='ISO-8859-5')
reader = mlio.CsvReader(rdr_prm, csv_params)
example = reader.read_example()
nonutf_feature = example['col_3']
try:
feature_np = as_numpy(nonutf_feature)
except SystemError as err:
pytest.fail("Unexpected exception thrown")
def _get_csv_dmatrix_pipe_mode(pipe_path, csv_weights):
"""Get Data Matrix from CSV data in pipe mode.
:param pipe_path: SageMaker pipe path where CSV formatted training data is piped
:param csv_weights: 1 if instance weights are in second column of CSV data; else 0
:return: xgb.DMatrix or None
"""
try:
pipes_path = pipe_path if isinstance(pipe_path, list) else [pipe_path]
dataset = [mlio.SageMakerPipe(path) for path in pipes_path]
reader_params = mlio.DataReaderParams(dataset=dataset,
batch_size=BATCH_SIZE)
csv_params = mlio.CsvParams(header_row_index=None)
reader = mlio.CsvReader(reader_params, csv_params)
# Check if data is present in reader
if reader.peek_example() is not None:
examples = []
for example in reader:
# Write each feature (column) of example into a single numpy array
tmp = [as_numpy(feature).squeeze() for feature in example]
tmp = np.array(tmp)
if len(tmp.shape) > 1:
# Columns are written as rows, needs to be transposed
tmp = tmp.T
else:
# If tmp is a 1-D array, it needs to be reshaped as a matrix
tmp = np.reshape(tmp, (1, tmp.shape[0]))
examples.append(tmp)
data = np.vstack(examples)
del examples
if csv_weights == 1:
dmatrix = xgb.DMatrix(data[:, 2:],
label=data[:, 0],
weight=data[:, 1])
else:
dmatrix = xgb.DMatrix(data[:, 1:], label=data[:, 0])
return dmatrix
else:
return None
except Exception as e:
raise exc.UserError(
"Failed to load csv data with exception:\n{}".format(e))
async def __train(self):
chunk_size = self.params.chunk_size # getattr(self.params, "chunk_size")
dataset = mlio.list_files(getattr(self.params, "dataset_path"),
pattern='*.csv')
logging.debug('mlio dataset={}'.format(dataset))
reader_params = mlio.DataReaderParams(
dataset=dataset,
batch_size=chunk_size,
num_prefetched_batches=self.params.num_prefetched_chunks)
reader = mlio.CsvReader(reader_params)
logging.debug('mlio reader={}'.format(reader))
num_epochs = self.params.num_epochs # Number of times to read the full dataset.
# use eta parameteres
eta = 0.01
if self.params.ml_lib == 'snap':
eta = 0.1
from pai4sk import BoostingMachine as Booster
else:
from sklearn.tree import DecisionTreeRegressor
logging.debug('starting training')
models = []
# preample
chunkim1 = reader.read_example()
if chunkim1 != None:
X_im1, y_im1 = await self.__preprocess_chunk(chunkim1)
chunki = reader.read_example()
i = 1
logging.debug('chunk{}={}'.format(0, chunkim1))
logging.debug('chunk{}={}'.format(i, chunki))
while chunki != None:
logging.debug('chunk{}={}'.format(i, chunki))
task_preprocess = asyncio.create_task(
self.__preprocess_chunk(chunki))
task_train = asyncio.create_task(self.__train_chunk(X_im1, y_im1))
X_i, y_i = await task_preprocess
models.extend(await task_train)
X_im1 = X_i
y_im1 = y_i
chunkim1 = chunki
chunki = reader.read_example()
i += 1
# postample
if chunkim1 != None:
logging.debug('y{}m1={}'.format(i, y_im1))
models.extend(await self.__train_chunk(X_im1, y_im1))
return models
def test_data_reader_params_members():
filename = os.path.join(resources_dir, 'test.pbr')
dataset = [mlio.File(filename)]
rdr_prm = mlio.DataReaderParams(dataset=dataset, batch_size=1)
assert rdr_prm.dataset == dataset
assert rdr_prm.batch_size == 1
assert rdr_prm.num_prefetched_batches == 0
assert rdr_prm.num_parallel_reads == 0
assert rdr_prm.last_batch_handling == mlio.LastBatchHandling.NONE
assert rdr_prm.bad_batch_handling == mlio.BadBatchHandling.ERROR
assert rdr_prm.num_instances_to_skip == 0
assert rdr_prm.num_instances_to_read is None
assert rdr_prm.shard_index == 0
assert rdr_prm.num_shards == 0
assert rdr_prm.shuffle_instances is False
assert rdr_prm.shuffle_window == 0
assert rdr_prm.shuffle_seed is None
assert rdr_prm.reshuffle_each_epoch is True
assert rdr_prm.subsample_ratio is None
rdr_prm.batch_size = 2
assert rdr_prm.batch_size == 2
