def my_pipeline():
spec = {
"spark_version": "5.3.x-scala2.11",
"spark_conf": {
"spark.speculation": "true"
}
}
expected_spec = {
"cluster_name": "test-cluster",
"spark_version": "5.3.x-scala2.11",
"node_type_id": "Standard_D3_v2",
"spark_conf": {
"spark.speculation": "true"
},
"num_workers": 2
}
res = CreateClusterOp(name="createcluster",
cluster_name="test-cluster",
spec=spec,
node_type_id="Standard_D3_v2",
num_workers=2)
self.assert_res(res, expected_spec)
def my_pipeline():
cluster_name = "test-cluster"
json_spec = """
{
"spark_version": "5.3.x-scala2.11",
"node_type_id": "Standard_D3_v2",
"spark_conf": {
"spark.speculation": "true"
},
"num_workers": 2
}
"""
expected_spec = {
"cluster_name": cluster_name,
"spark_version": "5.3.x-scala2.11",
"node_type_id": "Standard_D3_v2",
"spark_conf": {
"spark.speculation": "true"
},
"num_workers": 2
}
res = CreateClusterOp.from_json_spec(name="createcluster",
cluster_name=cluster_name,
json_spec=json_spec)
self.assert_res(res, expected_spec)
def my_pipeline():
cluster_name = "test-cluster"
spark_version = "5.3.x-scala2.11"
node_type_id = "Standard_D3_v2"
autoscale = {
"min_workers": 2,
"max_workers": 50
}
expected_spec = {
"cluster_name": cluster_name,
"spark_version":spark_version,
"node_type_id": node_type_id,
"autoscale": autoscale
}
res = CreateClusterOp(
name="createcluster",
cluster_name=cluster_name,
spark_version=spark_version,
node_type_id=node_type_id,
autoscale=autoscale
)
self.assert_res(res, expected_spec)
def my_pipeline():
cluster_name = "test-cluster"
spark_version = "5.3.x-scala2.11"
node_type_id = "Standard_D3_v2"
spark_conf = {
"spark.speculation": "true"
}
num_workers = 2
expected_spec = {
"cluster_name": cluster_name,
"spark_version": spark_version,
"node_type_id": node_type_id,
"spark_conf": spark_conf,
"num_workers": num_workers
}
res = CreateClusterOp(
name="createcluster",
cluster_name=cluster_name,
spark_version=spark_version,
node_type_id=node_type_id,
spark_conf=spark_conf,
num_workers=num_workers
)
self.assert_res(res, expected_spec)
def my_pipeline():
CreateClusterOp(
name="createcluster",
spark_version="5.3.x-scala2.11",
node_type_id="Standard_D3_v2",
spark_conf={
"spark.speculation": "true"
},
num_workers=2
)
def my_pipeline():
cluster_name = "test-cluster"
current_path = Path(__file__).parent
json_spec_file_name = current_path.joinpath("cluster_spec.json")
expected_spec = {
"cluster_name": cluster_name,
"spark_version": "5.3.x-scala2.11",
"node_type_id": "Standard_D3_v2",
"spark_conf": {
"spark.speculation": "true"
},
"num_workers": 2
}
res = CreateClusterOp.from_file_name(name="createcluster",
cluster_name=cluster_name,
file_name=json_spec_file_name)
self.assert_res(res, expected_spec)