def test_list_jobs(self):
# Setup Expected Response
next_page_token = ""
jobs_element = {}
jobs = [jobs_element]
expected_response = {"next_page_token": next_page_token, "jobs": jobs}
expected_response = jobs_pb2.ListJobsResponse(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = dataproc_v1beta2.JobControllerClient()
# Setup Request
project_id = "projectId-1969970175"
region = "region-934795532"
paged_list_response = client.list_jobs(project_id, region)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.jobs[0] == resources[0]
assert len(channel.requests) == 1
expected_request = jobs_pb2.ListJobsRequest(
project_id=project_id, region=region
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_submit_job(self):
# Setup Expected Response
driver_output_resource_uri = 'driverOutputResourceUri-542229086'
driver_control_files_uri = 'driverControlFilesUri207057643'
expected_response = {
'driver_output_resource_uri': driver_output_resource_uri,
'driver_control_files_uri': driver_control_files_uri
}
expected_response = jobs_pb2.Job(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
client = dataproc_v1beta2.JobControllerClient(channel=channel)
# Setup Request
project_id = 'projectId-1969970175'
region = 'region-934795532'
job = {}
response = client.submit_job(project_id, region, job)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = jobs_pb2.SubmitJobRequest(project_id=project_id,
region=region,
job=job)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_jobs(self):
# Setup Expected Response
next_page_token = ''
jobs_element = {}
jobs = [jobs_element]
expected_response = {'next_page_token': next_page_token, 'jobs': jobs}
expected_response = jobs_pb2.ListJobsResponse(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
client = dataproc_v1beta2.JobControllerClient(channel=channel)
# Setup Request
project_id = 'projectId-1969970175'
region = 'region-934795532'
paged_list_response = client.list_jobs(project_id, region)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.jobs[0] == resources[0]
assert len(channel.requests) == 1
expected_request = jobs_pb2.ListJobsRequest(project_id=project_id,
region=region)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_submit_job(self):
# Setup Expected Response
submitted_by = "submittedBy-2047729125"
driver_output_resource_uri = "driverOutputResourceUri-542229086"
driver_control_files_uri = "driverControlFilesUri207057643"
job_uuid = "jobUuid-1615012099"
expected_response = {
"submitted_by": submitted_by,
"driver_output_resource_uri": driver_output_resource_uri,
"driver_control_files_uri": driver_control_files_uri,
"job_uuid": job_uuid,
}
expected_response = jobs_pb2.Job(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = dataproc_v1beta2.JobControllerClient()
# Setup Request
project_id = "projectId-1969970175"
region = "region-934795532"
job = {}
response = client.submit_job(project_id, region, job)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = jobs_pb2.SubmitJobRequest(
project_id=project_id, region=region, job=job
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_jobs_exception(self):
channel = ChannelStub(responses=[CustomException()])
client = dataproc_v1beta2.JobControllerClient(channel=channel)
# Setup request
project_id = 'projectId-1969970175'
region = 'region-934795532'
paged_list_response = client.list_jobs(project_id, region)
with pytest.raises(CustomException):
list(paged_list_response)
def test_submit_job_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
client = dataproc_v1beta2.JobControllerClient(channel=channel)
# Setup request
project_id = 'projectId-1969970175'
region = 'region-934795532'
job = {}
with pytest.raises(CustomException):
client.submit_job(project_id, region, job)
def test_list_jobs_exception(self):
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = dataproc_v1beta2.JobControllerClient()
# Setup request
project_id = "projectId-1969970175"
region = "region-934795532"
paged_list_response = client.list_jobs(project_id, region)
with pytest.raises(CustomException):
list(paged_list_response)
def test_submit_job_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = dataproc_v1beta2.JobControllerClient()
# Setup request
project_id = "projectId-1969970175"
region = "region-934795532"
job = {}
with pytest.raises(CustomException):
client.submit_job(project_id, region, job)
def test_delete_job(self):
channel = ChannelStub()
client = dataproc_v1beta2.JobControllerClient(channel=channel)
# Setup Request
project_id = 'projectId-1969970175'
region = 'region-934795532'
job_id = 'jobId-1154752291'
client.delete_job(project_id, region, job_id)
assert len(channel.requests) == 1
expected_request = jobs_pb2.DeleteJobRequest(
project_id=project_id, region=region, job_id=job_id)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_delete_job(self):
channel = ChannelStub()
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = dataproc_v1beta2.JobControllerClient()
# Setup Request
project_id = "projectId-1969970175"
region = "region-934795532"
job_id = "jobId-1154752291"
client.delete_job(project_id, region, job_id)
assert len(channel.requests) == 1
expected_request = jobs_pb2.DeleteJobRequest(
project_id=project_id, region=region, job_id=job_id
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def __init__(self, project_id, cluster_name, bucket_name):
self.credentials = credentials
self.zone = 'europe-west6-b'
self.project_id = project_id
self.cluster_name = cluster_name
self.bucket_name = bucket_name
self.region = self._get_region_from_zone()
# Use a regional gRPC endpoint. See:
# https://cloud.google.com/dataproc/docs/concepts/regional-endpoints
self.client_transport = \
cluster_controller_grpc_transport.ClusterControllerGrpcTransport(credentials=self.credentials,
address='{}-dataproc.googleapis.com:443' \
.format(self.region))
self.job_transport = \
job_controller_grpc_transport.JobControllerGrpcTransport(credentials=self.credentials,
address='{}-dataproc.googleapis.com:443'.format(
self.region))
self.dataproc_cluster_client = dataproc_v1beta2.ClusterControllerClient(
self.client_transport)
self.dataproc_job_client = dataproc_v1beta2.JobControllerClient(
self.job_transport)
from google.cloud import dataproc_v1beta2
from time import sleep
clusterClient = dataproc_v1beta2.ClusterControllerClient()
jobClient = dataproc_v1beta2.JobControllerClient()
# T_odo: Initialize `project_id`:
project_id = 'PROJECT-ID'
# T_odo: Initialize `region`:
region = 'global'
# T_odo: Initialize `cluster`:
cluster = {
"cluster_name": "CLUSTER-NAME",
"config": {
"config_bucket": "",
"gce_cluster_config": {
"zone_uri": "ZONE-URI",
"metadata": {},
"subnetwork_uri": "default"
},
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
"disk_config": {
"boot_disk_type": "pd-standard",
"boot_disk_size_gb": 100,
"num_local_ssds": 0
}
},