from tfx.dsl.components.base import executor_spec
from tfx.extensions.google_cloud_ai_platform.trainer import \
executor as ai_platform_trainer_executor
from tfx.extensions.google_cloud_ai_platform.trainer.executor \
import TRAINING_ARGS_KEY, JOB_ID_KEY
from zenml.core.backends.training.training_base_backend import \
TrainingBaseBackend
from zenml.core.pipelines.utils import sanitize_name_for_ai_platform
from zenml.utils import requirement_utils
from zenml.utils.constants import ZENML_TRAINER_IMAGE_NAME
from zenml.utils.enums import GCPGPUTypes
from zenml.utils.logger import get_logger
requirement_utils.check_integration(requirement_utils.GCP_INTEGRATION)
logger = get_logger(__name__)
class SingleGPUTrainingGCAIPBackend(TrainingBaseBackend):
"""
Runs a TrainerStep on Google Cloud AI Platform.
A training backend can be used to efficiently train a machine learning
model on large amounts of data. This triggers a Training job on the Google
Cloud AI Platform service: https://cloud.google.com/ai-platform.
This backend is meant for a training job with a single GPU only. The user
has a choice of three GPUs, specified in the GCPGPUTypes Enum.
"""
# Copyright (c) maiot GmbH 2020. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from zenml.core.steps.trainer.base_trainer import BaseTrainerStep
from zenml.utils import requirement_utils
requirement_utils.check_integration(requirement_utils.PYTORCH_INTEGRATION)
class TorchBaseTrainerStep(BaseTrainerStep):
"""
Base class for all PyTorch based trainer steps. All pytorch based
trainings should use this as the base class. An example is available
with torch_ff_trainer.FeedForwardTrainer.
"""
pass
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from zenml.steps.data.base_data_step import BaseDataStep
from zenml.steps.data.bq_data_step import BQDataStep
from zenml.steps.data.csv_data_step import CSVDataStep
from zenml.steps.data.image_data_step import ImageDataStep
from zenml.logger import get_logger
from zenml.utils.requirement_utils import check_integration, \
POSTGRES_INTEGRATION
logger = get_logger(__name__)
try:
check_integration(POSTGRES_INTEGRATION)
from zenml.steps.data.postgres_data_step import PostgresDataStep
except ModuleNotFoundError as e:
logger.debug(f"There were failed imports due to missing integrations. "
f"PostgresDataStep was not imported. "
f"More information:")
logger.debug(e)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from zenml.steps.tokenizer.base_tokenizer import BaseTokenizer
from zenml.logger import get_logger
from zenml.utils.requirement_utils import check_integration, \
HUGGINGFACE_INTEGRATION
logger = get_logger(__name__)
# HuggingFace extra requirement
try:
check_integration(HUGGINGFACE_INTEGRATION)
from zenml.steps.tokenizer.hf_tokenizer import \
HuggingFaceTokenizerStep
except ModuleNotFoundError as e:
logger.debug(f"There were failed imports due to missing integrations. "
f"HuggingFaceTokenizerStep was not imported. "
f"More information:")
logger.debug(e)
# Copyright (c) maiot GmbH 2020. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from zenml.steps.deployer.base_deployer import BaseDeployerStep
from zenml.steps.deployer.gcaip_deployer import GCAIPDeployer
from zenml.logger import get_logger
from zenml.utils.requirement_utils import check_integration, \
CORTEX_INTEGRATION
logger = get_logger(__name__)
try:
check_integration(CORTEX_INTEGRATION)
from zenml.steps.deployer.cortex_deployer import CortexDeployer
except ModuleNotFoundError as e:
logger.debug(f"There were failed imports due to missing integrations. "
f"HuggingFaceTokenizerStep was not imported. "
f"More information:")
logger.debug(e)
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from zenml.steps.trainer.base_trainer import BaseTrainerStep
from zenml.steps.trainer.tensorflow_trainers.tf_base_trainer import \
TFBaseTrainerStep
from zenml.steps.trainer.tensorflow_trainers.tf_ff_trainer import \
FeedForwardTrainer as TFFeedForwardTrainer
from zenml.logger import get_logger
from zenml.utils.requirement_utils import check_integration, \
PYTORCH_INTEGRATION
logger = get_logger(__name__)
# wrap Pytorch extra integrations safely
try:
check_integration(PYTORCH_INTEGRATION)
from zenml.steps.trainer.pytorch_trainers.torch_base_trainer import \
TorchBaseTrainerStep
from zenml.steps.trainer.pytorch_trainers.torch_ff_trainer import \
FeedForwardTrainer as TorchFeedForwardTrainer
except ModuleNotFoundError as e:
logger.debug(f"There were failed imports due to missing integrations. "
f"TorchBaseTrainerStep, TorchFeedForwardTrainer were not "
f"imported. More information:")
logger.debug(e)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from zenml.backends.training.training_base_backend import \
TrainingBaseBackend
from zenml.logger import get_logger
from zenml.utils.requirement_utils import check_integration, \
GCP_INTEGRATION
logger = get_logger(__name__)
try:
check_integration(GCP_INTEGRATION)
from zenml.backends.training.training_gcaip_backend import \
SingleGPUTrainingGCAIPBackend
except ModuleNotFoundError as e:
logger.debug(f"There were failed imports due to missing integrations. "
f"SingleGPUTrainingGCAIPBackend was not imported. "
f"More information:")
logger.debug(e)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
from zenml.backends.orchestrator.base.orchestrator_base_backend import \
OrchestratorBaseBackend
from zenml.backends.orchestrator.gcp.orchestrator_gcp_backend import \
OrchestratorGCPBackend
from zenml.backends.orchestrator.kubeflow.orchestrator_kubeflow_backend \
import OrchestratorKubeFlowBackend
from zenml.backends.orchestrator.kubernetes.orchestrator_kubernetes_backend \
import OrchestratorKubernetesBackend
from zenml.logger import get_logger
from zenml.utils.requirement_utils import check_integration, \
AWS_INTEGRATION
logger = get_logger(__name__)
# check extra AWS requirements
try:
check_integration(AWS_INTEGRATION)
from zenml.backends.orchestrator.aws.orchestrator_aws_backend import \
OrchestratorAWSBackend
except ModuleNotFoundError as e:
logger.debug(f"There were failed imports due to missing integrations. "
f"OrchestratorAWSBackend was not imported. "
f"More information:")
logger.debug(e)
