def __init__(self, **kwargs):
AbstractDataReader.__init__(self, **kwargs)
self._kwargs = kwargs
self._metadata = Metadata(column_names=None)
self._table = self._kwargs["table"]
self._columns = self._kwargs.get("columns")
self._init_metadata()
# Initialize an ODPS IO reader for each table with task type
self._table_readers = dict()
def test_odps_data_reader_integration_with_local_keras(self):
num_records = 2
model_spec = load_module(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"../../../model_zoo",
"odps_iris_dnn_model/odps_iris_dnn_model.py",
)
).__dict__
model = model_spec["custom_model"]()
optimizer = model_spec["optimizer"]()
loss = model_spec["loss"]
reader = create_data_reader(
data_origin=self.test_table,
records_per_task=10,
**{"columns": IRIS_TABLE_COLUMN_NAMES, "label_col": "class"}
)
dataset_fn = reader.default_dataset_fn()
def _gen():
for data in self.reader.read_records(
_MockedTask(
0,
num_records,
self.test_table + ":shard_0",
elasticdl_pb2.TRAINING,
)
):
if data is not None:
yield data
dataset = tf.data.Dataset.from_generator(_gen, tf.string)
dataset = dataset_fn(
dataset, None, Metadata(column_names=IRIS_TABLE_COLUMN_NAMES)
)
dataset = dataset.batch(1)
loss_history = []
grads = None
for features, labels in dataset:
with tf.GradientTape() as tape:
logits = model(features, training=True)
loss_value = loss(labels, logits)
loss_history.append(loss_value.numpy())
grads = tape.gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
self.assertEqual(len(loss_history), num_records)
self.assertEqual(len(grads), num_records)
self.assertEqual(len(model.trainable_variables), num_records)
def metadata(self):
return Metadata(column_names=self.selected_columns)
def metadata(self):
return Metadata(column_names=None)
def __init__(self, **kwargs):
AbstractDataReader.__init__(self, **kwargs)
self._kwargs = kwargs
self._metadata = Metadata(column_names=None)