def __init__(self,
app_name: str,
num_executors: int,
executor_cores: int,
executor_memory: Union[str, int],
configs: Dict[str, str] = None):
self._app_name = app_name
self._num_executors = num_executors
self._executor_cores = executor_cores
if isinstance(executor_memory, str):
# If this is human readable str(like: 10KB, 10MB..), parse it
executor_memory = parse_memory_size(executor_memory)
self._executor_memory = executor_memory
self._configs = {} if configs is None else configs
self._spark_cluster: Optional[SparkCluster] = None
self._spark_session: Optional[SparkSession] = None
def test_memory_size_parser():
upper_units = ["", "K", "M", "G", "T"]
expected = [10 * math.pow(2, 10 * p) for p in range(len(upper_units))]
# upper without B
values = [f"10{unit}" for unit in upper_units]
parsed = [utils.parse_memory_size(v) for v in values]
assert parsed == expected
# lower without B
values = [f"10{unit.lower()}" for unit in upper_units]
parsed = [utils.parse_memory_size(v) for v in values]
assert parsed == expected
# upper blank without B
values = [f"10 {unit}" for unit in upper_units]
parsed = [utils.parse_memory_size(v) for v in values]
assert parsed == expected
# upper two blanks without B
values = [f"10 {unit}" for unit in upper_units]
parsed = [utils.parse_memory_size(v) for v in values]
assert parsed == expected
upper_units = ["B", "KB", "MB", "GB", "TB"]
# upper with B
values = [f"10{unit}" for unit in upper_units]
parsed = [utils.parse_memory_size(v) for v in values]
assert parsed == expected
# lower with B
values = [f"10{unit.lower()}" for unit in upper_units]
parsed = [utils.parse_memory_size(v) for v in values]
assert parsed == expected
# upper blank with B
values = [f"10 {unit}" for unit in upper_units]
parsed = [utils.parse_memory_size(v) for v in values]
assert parsed == expected
# upper two blanks with B
values = [f"10 {unit}" for unit in upper_units]
parsed = [utils.parse_memory_size(v) for v in values]
assert parsed == expected