path='gs://zenml_quickstart/diabetes.csv')
except AlreadyExistsException:
ds = Repository.get_instance().get_datasource_by_name(
'Pima Indians Diabetes')
training_pipeline.add_datasource(ds)
# Add a split
training_pipeline.add_split(RandomSplit(split_map={'eval': 0.3, 'train': 0.7}))
# Add a preprocessing unit
training_pipeline.add_preprocesser(
StandardPreprocesser(features=[
'times_pregnant', 'pgc', 'dbp', 'tst', 'insulin', 'bmi', 'pedigree',
'age'
],
labels=['has_diabetes'],
overwrite={
'has_diabetes': {
'transform': [{
'method': 'no_transform',
'parameters': {}
}]
}
}))
# Add a trainer
training_pipeline.add_trainer(MyPyTorchLightningTrainer(epoch=100))
# Run the pipeline locally
training_pipeline.run()
training_pipeline.add_evaluator(
TFMAEvaluator(slices=[['has_diabetes']],
metrics={
transformed_label_name('has_diabetes'):
['binary_crossentropy', 'binary_accuracy']
}))
# Define the metadata store
metadata_store = MySQLMetadataStore(
host=MYSQL_HOST,
port=int(MYSQL_PORT),
database=MYSQL_DB,
username=MYSQL_USER,
password=MYSQL_PWD,
)
# Define the artifact store
artifact_store = ArtifactStore(
os.path.join(GCP_BUCKET, 'gcp_gcaip_training/artifact_store'))
# Define the orchestrator backend
orchestrator_backend = OrchestratorGCPBackend(
cloudsql_connection_name=GCP_CLOUD_SQL_INSTANCE_NAME, project=GCP_PROJECT)
# Run the pipeline
training_pipeline.run(
backend=orchestrator_backend,
metadata_store=metadata_store,
artifact_store=artifact_store,
)
overwrite={
'has_diabetes': {
'transform': [{
'method': 'no_transform',
'parameters': {}
}]
}
}).with_backend(processing_backend))
# Add a trainer
training_pipeline.add_trainer(
TFFeedForwardTrainer(loss='binary_crossentropy',
last_activation='sigmoid',
output_units=1,
metrics=['accuracy'],
epochs=20))
# Add an evaluator
training_pipeline.add_evaluator(
TFMAEvaluator(slices=[['has_diabetes']],
metrics={
transformed_label_name('has_diabetes'):
['binary_crossentropy', 'binary_accuracy']
}).with_backend(processing_backend))
# Define the artifact store
artifact_store = ArtifactStore(
os.path.join(GCP_BUCKET, 'dataflow_processing/artifact_store'))
# Run the pipeline
training_pipeline.run(artifact_store=artifact_store)
