def test_list_jobs(self): # Setup Expected Response next_page_token = "" jobs_element = {} jobs = [jobs_element] expected_response = {"next_page_token": next_page_token, "jobs": jobs} expected_response = jobs_pb2.ListJobsResponse(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = dataproc_v1beta2.JobControllerClient() # Setup Request project_id = "projectId-1969970175" region = "region-934795532" paged_list_response = client.list_jobs(project_id, region) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.jobs[0] == resources[0] assert len(channel.requests) == 1 expected_request = jobs_pb2.ListJobsRequest( project_id=project_id, region=region ) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_submit_job(self): # Setup Expected Response driver_output_resource_uri = 'driverOutputResourceUri-542229086' driver_control_files_uri = 'driverControlFilesUri207057643' expected_response = { 'driver_output_resource_uri': driver_output_resource_uri, 'driver_control_files_uri': driver_control_files_uri } expected_response = jobs_pb2.Job(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) client = dataproc_v1beta2.JobControllerClient(channel=channel) # Setup Request project_id = 'projectId-1969970175' region = 'region-934795532' job = {} response = client.submit_job(project_id, region, job) assert expected_response == response assert len(channel.requests) == 1 expected_request = jobs_pb2.SubmitJobRequest(project_id=project_id, region=region, job=job) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_jobs(self): # Setup Expected Response next_page_token = '' jobs_element = {} jobs = [jobs_element] expected_response = {'next_page_token': next_page_token, 'jobs': jobs} expected_response = jobs_pb2.ListJobsResponse(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) client = dataproc_v1beta2.JobControllerClient(channel=channel) # Setup Request project_id = 'projectId-1969970175' region = 'region-934795532' paged_list_response = client.list_jobs(project_id, region) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.jobs[0] == resources[0] assert len(channel.requests) == 1 expected_request = jobs_pb2.ListJobsRequest(project_id=project_id, region=region) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_submit_job(self): # Setup Expected Response submitted_by = "submittedBy-2047729125" driver_output_resource_uri = "driverOutputResourceUri-542229086" driver_control_files_uri = "driverControlFilesUri207057643" job_uuid = "jobUuid-1615012099" expected_response = { "submitted_by": submitted_by, "driver_output_resource_uri": driver_output_resource_uri, "driver_control_files_uri": driver_control_files_uri, "job_uuid": job_uuid, } expected_response = jobs_pb2.Job(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = dataproc_v1beta2.JobControllerClient() # Setup Request project_id = "projectId-1969970175" region = "region-934795532" job = {} response = client.submit_job(project_id, region, job) assert expected_response == response assert len(channel.requests) == 1 expected_request = jobs_pb2.SubmitJobRequest( project_id=project_id, region=region, job=job ) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_jobs_exception(self): channel = ChannelStub(responses=[CustomException()]) client = dataproc_v1beta2.JobControllerClient(channel=channel) # Setup request project_id = 'projectId-1969970175' region = 'region-934795532' paged_list_response = client.list_jobs(project_id, region) with pytest.raises(CustomException): list(paged_list_response)
def test_submit_job_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) client = dataproc_v1beta2.JobControllerClient(channel=channel) # Setup request project_id = 'projectId-1969970175' region = 'region-934795532' job = {} with pytest.raises(CustomException): client.submit_job(project_id, region, job)
def test_list_jobs_exception(self): channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = dataproc_v1beta2.JobControllerClient() # Setup request project_id = "projectId-1969970175" region = "region-934795532" paged_list_response = client.list_jobs(project_id, region) with pytest.raises(CustomException): list(paged_list_response)
def test_submit_job_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = dataproc_v1beta2.JobControllerClient() # Setup request project_id = "projectId-1969970175" region = "region-934795532" job = {} with pytest.raises(CustomException): client.submit_job(project_id, region, job)
def test_delete_job(self): channel = ChannelStub() client = dataproc_v1beta2.JobControllerClient(channel=channel) # Setup Request project_id = 'projectId-1969970175' region = 'region-934795532' job_id = 'jobId-1154752291' client.delete_job(project_id, region, job_id) assert len(channel.requests) == 1 expected_request = jobs_pb2.DeleteJobRequest( project_id=project_id, region=region, job_id=job_id) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_delete_job(self): channel = ChannelStub() patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = dataproc_v1beta2.JobControllerClient() # Setup Request project_id = "projectId-1969970175" region = "region-934795532" job_id = "jobId-1154752291" client.delete_job(project_id, region, job_id) assert len(channel.requests) == 1 expected_request = jobs_pb2.DeleteJobRequest( project_id=project_id, region=region, job_id=job_id ) actual_request = channel.requests[0][1] assert expected_request == actual_request
def __init__(self, project_id, cluster_name, bucket_name): self.credentials = credentials self.zone = 'europe-west6-b' self.project_id = project_id self.cluster_name = cluster_name self.bucket_name = bucket_name self.region = self._get_region_from_zone() # Use a regional gRPC endpoint. See: # https://cloud.google.com/dataproc/docs/concepts/regional-endpoints self.client_transport = \ cluster_controller_grpc_transport.ClusterControllerGrpcTransport(credentials=self.credentials, address='{}-dataproc.googleapis.com:443' \ .format(self.region)) self.job_transport = \ job_controller_grpc_transport.JobControllerGrpcTransport(credentials=self.credentials, address='{}-dataproc.googleapis.com:443'.format( self.region)) self.dataproc_cluster_client = dataproc_v1beta2.ClusterControllerClient( self.client_transport) self.dataproc_job_client = dataproc_v1beta2.JobControllerClient( self.job_transport)
from google.cloud import dataproc_v1beta2 from time import sleep clusterClient = dataproc_v1beta2.ClusterControllerClient() jobClient = dataproc_v1beta2.JobControllerClient() # T_odo: Initialize `project_id`: project_id = 'PROJECT-ID' # T_odo: Initialize `region`: region = 'global' # T_odo: Initialize `cluster`: cluster = { "cluster_name": "CLUSTER-NAME", "config": { "config_bucket": "", "gce_cluster_config": { "zone_uri": "ZONE-URI", "metadata": {}, "subnetwork_uri": "default" }, "master_config": { "num_instances": 1, "machine_type_uri": "n1-standard-4", "disk_config": { "boot_disk_type": "pd-standard", "boot_disk_size_gb": 100, "num_local_ssds": 0 } },