def run_fn(self):
feature_spec = get_feature_spec_from_schema(self.schema_path)
train_dataset = self.input_fn(self.train_files, feature_spec)
eval_dataset = self.input_fn(self.eval_files, feature_spec)
model = self.model_fn(train_dataset=train_dataset,
eval_dataset=eval_dataset)
model.save_pretrained(self.serving_model_dir, saved_model=True)
def get_predictions(self, sample_size: int = 100000):
"""
Samples prediction data as a pandas DataFrame.
Args:
sample_size: # of rows to sample.
"""
base_uri = self.get_artifacts_uri_by_component(
GDPComponent.Inferrer.name)[0]
data_files = path_utils.list_dir(os.path.join(base_uri, 'examples'))
dataset = tf.data.TFRecordDataset(data_files, compression_type='GZIP')
schema_uri = self.get_artifacts_uri_by_component(
GDPComponent.DataSchema.name)[0]
spec = get_feature_spec_from_schema(schema_uri)
return convert_raw_dataset_to_pandas(dataset, spec, sample_size)
def sample_data(self, sample_size: int = 100000):
"""
Sampels data from datasource as a pandas DataFrame.
Args:
sample_size: # of rows to sample.
"""
pipeline = self._get_one_pipeline()
data_files = self._get_data_file_paths(pipeline)
schema_uri = pipeline.get_artifacts_uri_by_component(
GDPComponent.DataSchema.name)[0]
spec = get_feature_spec_from_schema(schema_uri)
dataset = tf.data.TFRecordDataset(data_files, compression_type='GZIP')
return convert_raw_dataset_to_pandas(dataset, spec, sample_size)
def sample_transformed_data(self,
split_name: Text = 'eval',
sample_size: int = 100000):
"""
Samples transformed data as a pandas DataFrame.
Args:
split_name: name of split to see
sample_size: # of rows to sample.
"""
base_uri = self.get_artifacts_uri_by_component(
GDPComponent.Transform.name)[0]
transform_schema = os.path.join(base_uri, 'transformed_metadata')
spec = get_feature_spec_from_schema(transform_schema)
base_uri = Path(base_uri)
id_ = base_uri.name
transform_data_path = os.path.join(str(base_uri.parent.parent),
'transformed_examples', id_)
split_data_path = os.path.join(transform_data_path, split_name)
data_files = path_utils.list_dir(split_data_path)
dataset = tf.data.TFRecordDataset(data_files, compression_type='GZIP')
return convert_raw_dataset_to_pandas(dataset, spec, sample_size)