def create( self, cluster_identifier: ClusterIdentifier) -> DataprocClusterManager: """Creates a Dataproc cluster manager provisioned for the cluster identified. If the cluster is known, returns an existing cluster manager. """ # Try to get some not-None cluster metadata. cluster_metadata = self.cluster_metadata(cluster_identifier) if not cluster_metadata: raise ValueError( 'Unknown cluster identifier: %s. Cannot create or reuse' 'a Dataproc cluster.') elif cluster_metadata.region == 'global': # The global region is unsupported as it will be eventually deprecated. raise ValueError( 'Clusters in the global region are not supported.') elif not cluster_metadata.region: _LOGGER.info( 'No region information was detected, defaulting Dataproc cluster ' 'region to: us-central1.') cluster_metadata.region = 'us-central1' # else use the provided region. known_dcm = self.dataproc_cluster_managers.get(cluster_metadata, None) if known_dcm: return known_dcm dcm = DataprocClusterManager(cluster_metadata) dcm.create_flink_cluster() # ClusterMetadata with derivative fields populated by the dcm. derived_meta = dcm.cluster_metadata self.dataproc_cluster_managers[derived_meta] = dcm self.master_urls[derived_meta.master_url] = derived_meta # Update the default cluster metadata to the one just created. self.set_default_cluster(derived_meta) return dcm
def _create_dataproc_cluster_if_applicable(self, user_pipeline): """ Creates a Dataproc cluster if the provided user_pipeline is running FlinkRunner and no flink_master_url was provided as an option. A cluster is not created when a flink_master_url is detected. Example pipeline options to enable automatic Dataproc cluster creation: options = PipelineOptions([ '--runner=FlinkRunner', '--project=my-project', '--region=my-region', '--environment_type=DOCKER' ]) Example pipeline options to skip automatic Dataproc cluster creation: options = PipelineOptions([ '--runner=FlinkRunner', '--flink_master=example.internal:41979', '--environment_type=DOCKER' ]) """ from apache_beam.runners.portability.flink_runner import FlinkRunner from apache_beam.options.pipeline_options import FlinkRunnerOptions flink_master = user_pipeline.options.view_as( FlinkRunnerOptions).flink_master clusters = ie.current_env().clusters # Only consider this logic when both below 2 conditions apply. if isinstance(self._underlying_runner, FlinkRunner) and clusters.dataproc_cluster_managers.get( str(id(user_pipeline)), None) is None: if flink_master == '[auto]': # The above condition is True when the user has not provided a # flink_master. if ie.current_env()._is_in_ipython: warnings.filterwarnings( 'ignore', 'options is deprecated since First stable release. References to ' '<pipeline>.options will not be supported', category=DeprecationWarning) project_id = ( user_pipeline.options.view_as(GoogleCloudOptions).project) region = ( user_pipeline.options.view_as(GoogleCloudOptions).region) cluster_name = ie.current_env().clusters.default_cluster_name cluster_metadata = MasterURLIdentifier( project_id=project_id, region=region, cluster_name=cluster_name) else: cluster_metadata = clusters.master_urls.inverse.get( flink_master, None) # else noop, no need to log anything because we allow a master_url # (not managed by us) provided by the user. if cluster_metadata: # create the cluster_manager and populate dicts in the clusters # instance if the pipeline is not already mapped to an existing # cluster_manager. cluster_manager = DataprocClusterManager(cluster_metadata) cluster_manager.create_flink_cluster() clusters.master_urls[ cluster_manager.master_url] = cluster_metadata clusters.dataproc_cluster_managers[str( id(user_pipeline))] = cluster_manager clusters.master_urls_to_pipelines[ cluster_manager.master_url].append(str(id(user_pipeline)))