from tfx.dsl.components.base import executor_spec from tfx.extensions.google_cloud_ai_platform.trainer import \ executor as ai_platform_trainer_executor from tfx.extensions.google_cloud_ai_platform.trainer.executor \ import TRAINING_ARGS_KEY, JOB_ID_KEY from zenml.core.backends.training.training_base_backend import \ TrainingBaseBackend from zenml.core.pipelines.utils import sanitize_name_for_ai_platform from zenml.utils import requirement_utils from zenml.utils.constants import ZENML_TRAINER_IMAGE_NAME from zenml.utils.enums import GCPGPUTypes from zenml.utils.logger import get_logger requirement_utils.check_integration(requirement_utils.GCP_INTEGRATION) logger = get_logger(__name__) class SingleGPUTrainingGCAIPBackend(TrainingBaseBackend): """ Runs a TrainerStep on Google Cloud AI Platform. A training backend can be used to efficiently train a machine learning model on large amounts of data. This triggers a Training job on the Google Cloud AI Platform service: https://cloud.google.com/ai-platform. This backend is meant for a training job with a single GPU only. The user has a choice of three GPUs, specified in the GCPGPUTypes Enum. """
# Copyright (c) maiot GmbH 2020. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. from zenml.core.steps.trainer.base_trainer import BaseTrainerStep from zenml.utils import requirement_utils requirement_utils.check_integration(requirement_utils.PYTORCH_INTEGRATION) class TorchBaseTrainerStep(BaseTrainerStep): """ Base class for all PyTorch based trainer steps. All pytorch based trainings should use this as the base class. An example is available with torch_ff_trainer.FeedForwardTrainer. """ pass
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. from zenml.steps.data.base_data_step import BaseDataStep from zenml.steps.data.bq_data_step import BQDataStep from zenml.steps.data.csv_data_step import CSVDataStep from zenml.steps.data.image_data_step import ImageDataStep from zenml.logger import get_logger from zenml.utils.requirement_utils import check_integration, \ POSTGRES_INTEGRATION logger = get_logger(__name__) try: check_integration(POSTGRES_INTEGRATION) from zenml.steps.data.postgres_data_step import PostgresDataStep except ModuleNotFoundError as e: logger.debug(f"There were failed imports due to missing integrations. " f"PostgresDataStep was not imported. " f"More information:") logger.debug(e)
# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. from zenml.steps.tokenizer.base_tokenizer import BaseTokenizer from zenml.logger import get_logger from zenml.utils.requirement_utils import check_integration, \ HUGGINGFACE_INTEGRATION logger = get_logger(__name__) # HuggingFace extra requirement try: check_integration(HUGGINGFACE_INTEGRATION) from zenml.steps.tokenizer.hf_tokenizer import \ HuggingFaceTokenizerStep except ModuleNotFoundError as e: logger.debug(f"There were failed imports due to missing integrations. " f"HuggingFaceTokenizerStep was not imported. " f"More information:") logger.debug(e)
# Copyright (c) maiot GmbH 2020. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. from zenml.steps.deployer.base_deployer import BaseDeployerStep from zenml.steps.deployer.gcaip_deployer import GCAIPDeployer from zenml.logger import get_logger from zenml.utils.requirement_utils import check_integration, \ CORTEX_INTEGRATION logger = get_logger(__name__) try: check_integration(CORTEX_INTEGRATION) from zenml.steps.deployer.cortex_deployer import CortexDeployer except ModuleNotFoundError as e: logger.debug(f"There were failed imports due to missing integrations. " f"HuggingFaceTokenizerStep was not imported. " f"More information:") logger.debug(e)
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. from zenml.steps.trainer.base_trainer import BaseTrainerStep from zenml.steps.trainer.tensorflow_trainers.tf_base_trainer import \ TFBaseTrainerStep from zenml.steps.trainer.tensorflow_trainers.tf_ff_trainer import \ FeedForwardTrainer as TFFeedForwardTrainer from zenml.logger import get_logger from zenml.utils.requirement_utils import check_integration, \ PYTORCH_INTEGRATION logger = get_logger(__name__) # wrap Pytorch extra integrations safely try: check_integration(PYTORCH_INTEGRATION) from zenml.steps.trainer.pytorch_trainers.torch_base_trainer import \ TorchBaseTrainerStep from zenml.steps.trainer.pytorch_trainers.torch_ff_trainer import \ FeedForwardTrainer as TorchFeedForwardTrainer except ModuleNotFoundError as e: logger.debug(f"There were failed imports due to missing integrations. " f"TorchBaseTrainerStep, TorchFeedForwardTrainer were not " f"imported. More information:") logger.debug(e)
# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. from zenml.backends.training.training_base_backend import \ TrainingBaseBackend from zenml.logger import get_logger from zenml.utils.requirement_utils import check_integration, \ GCP_INTEGRATION logger = get_logger(__name__) try: check_integration(GCP_INTEGRATION) from zenml.backends.training.training_gcaip_backend import \ SingleGPUTrainingGCAIPBackend except ModuleNotFoundError as e: logger.debug(f"There were failed imports due to missing integrations. " f"SingleGPUTrainingGCAIPBackend was not imported. " f"More information:") logger.debug(e)
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. from zenml.backends.orchestrator.base.orchestrator_base_backend import \ OrchestratorBaseBackend from zenml.backends.orchestrator.gcp.orchestrator_gcp_backend import \ OrchestratorGCPBackend from zenml.backends.orchestrator.kubeflow.orchestrator_kubeflow_backend \ import OrchestratorKubeFlowBackend from zenml.backends.orchestrator.kubernetes.orchestrator_kubernetes_backend \ import OrchestratorKubernetesBackend from zenml.logger import get_logger from zenml.utils.requirement_utils import check_integration, \ AWS_INTEGRATION logger = get_logger(__name__) # check extra AWS requirements try: check_integration(AWS_INTEGRATION) from zenml.backends.orchestrator.aws.orchestrator_aws_backend import \ OrchestratorAWSBackend except ModuleNotFoundError as e: logger.debug(f"There were failed imports due to missing integrations. " f"OrchestratorAWSBackend was not imported. " f"More information:") logger.debug(e)