def test_submit_job_error(self): http_mocked = HttpMockSequence([ ({ 'status': '200' }, open('tests/fixtures/dataproc/first_request.json', 'rb').read()), ({ 'status': '200' }, 'echo_request_body'), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_running.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_error.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_done.json', 'rb').read()) ]) dataproc_test = dataproc.Dataproc(project='project', region='region', http=http_mocked) with self.assertRaises(Exception): dataproc_test.submit_job("", "", [""], "", [])
def test_create_cluster(self): http_mocked = HttpMockSequence([ ({ 'status': '200' }, open('tests/fixtures/dataproc/first_request.json', 'rb').read()), ({ 'status': '200' }, 'echo_request_body'), ({ 'status': '200' }, open('tests/fixtures/dataproc/cluster_not_done.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/cluster_not_done.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/cluster_done.json', 'rb').read()) ]) dataproc_test = dataproc.Dataproc(project="project", region="region", http=http_mocked) result = dataproc_test.create_cluster("NAME", 2, ["B1", "B2"]) expected = { 'clusterName': 'NAME', 'projectId': 'project', 'config': { 'workerConfig': { 'machineTypeUri': 'n1-standard-4', 'numInstances': 2, 'instanceNames': ['B1', 'B2'], 'diskConfig': { 'numLocalSsds': 0, 'bootDiskSizeGb': 10 } }, 'masterConfig': { 'machineTypeUri': 'n1-standard-4', 'numInstances': 1, 'instanceNames': ['cluster-yarn-recsys-m'], 'diskConfig': { 'numLocalSsds': 0, 'bootDiskSizeGb': 10 } }, 'gceClusterConfig': { 'subnetworkUri': 'default', 'zoneUri': 'region-b' }, 'configBucket': '', 'softwareConfig': { 'imageVersion': '1.2.54-deb8' }, }, } self.assertEqual(result, expected)
def test_submit_job_spark_with_properties_successfully(self): self.maxDiff = None http_mocked = HttpMockSequence([ ({ 'status': '200' }, open('tests/fixtures/dataproc/first_request.json', 'rb').read()), ({ 'status': '200' }, 'echo_request_body'), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_running.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_running.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_done.json', 'rb').read()) ]) dataproc_test = dataproc.Dataproc(project='project', region='region', http=http_mocked) result = dataproc_test.submit_spark_job( 'CLUSTER', 'BUCKET', ['arg1', 'arg2'], ['/path/to/jar/jarname.jar'], 'main', properties={'a_property': 'a_property_value'}) body_request_expected = { 'projectId': 'project', 'job': { 'placement': { 'clusterName': 'CLUSTER' }, 'sparkJob': { 'jarFileUris': ['/path/to/jar/jarname.jar'], 'mainClass': 'main', 'args': ['arg1', 'arg2'], 'properties': { 'a_property': 'a_property_value' } }, 'reference': { 'jobId': 'spark_main_1994_04_27_12_00_01' } } } self.assertDictEqual(body_request_expected, result)
def test_submit_job_pyspark_with_properties_successfully(self): self.maxDiff = None http_mocked = HttpMockSequence([ ({ 'status': '200' }, open('tests/fixtures/dataproc/first_request.json', 'rb').read()), ({ 'status': '200' }, 'echo_request_body'), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_running.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_running.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/job_status_done.json', 'rb').read()) ]) dataproc_test = dataproc.Dataproc(project='project', region='region', http=http_mocked) result = dataproc_test.submit_pyspark_job( 'CLUSTER', 'BUCKET', ['arg1', 'arg2'], 'path/to/main.py', ['/path/to/py/file_one.py', '/path/to/py/file_two.py']) body_request_expected = { 'projectId': 'project', 'job': { 'placement': { 'clusterName': 'CLUSTER' }, 'pysparkJob': { 'pythonFileUris': ['/path/to/py/file_one.py', '/path/to/py/file_two.py'], 'mainPythonFileUri': 'gs://BUCKET/path/to/main.py', 'args': ['arg1', 'arg2'] }, 'reference': { 'jobId': 'pyspark_main_py_1994_04_27_12_00_01' } } } self.assertDictEqual(body_request_expected, result)
def test_list_cluster_1(self): http_mocked = HttpMockSequence([ ({ 'status': '200' }, open('tests/fixtures/dataproc/first_request.json', 'rb').read()), ({ 'status': '200' }, open('tests/fixtures/dataproc/list_clusters.json', 'rb').read()) ]) dataproc_test = dataproc.Dataproc(project="", region="", http=http_mocked) result = dataproc_test.list_clusters() self.assertEqual(len(result), 1)
def test_delete_cluster(self): http_mocked = HttpMockSequence([ ({ 'status': '200' }, open('tests/fixtures/dataproc/first_request.json', 'rb').read()), ({ 'status': '200' }, 'echo_request_headers_as_json'), ({ 'status': '200' }, open('tests/fixtures/dataproc/cluster_deleting.json', 'rb').read()) ]) dataproc_test = dataproc.Dataproc(project="project", region="region", http=http_mocked) result = dataproc_test.delete_cluster("NAME") self.assertEqual(result['content-length'], '0')