Exemplo n.º 1
0
    def __init__(self,
                 app_name: str,
                 num_executors: int,
                 executor_cores: int,
                 executor_memory: Union[str, int],
                 configs: Dict[str, str] = None):
        self._app_name = app_name
        self._num_executors = num_executors
        self._executor_cores = executor_cores

        if isinstance(executor_memory, str):
            # If this is human readable str(like: 10KB, 10MB..), parse it
            executor_memory = parse_memory_size(executor_memory)

        self._executor_memory = executor_memory
        self._configs = {} if configs is None else configs

        self._spark_cluster: Optional[SparkCluster] = None
        self._spark_session: Optional[SparkSession] = None
Exemplo n.º 2
0
def test_memory_size_parser():
    upper_units = ["", "K", "M", "G", "T"]
    expected = [10 * math.pow(2, 10 * p) for p in range(len(upper_units))]

    # upper without B
    values = [f"10{unit}" for unit in upper_units]
    parsed = [utils.parse_memory_size(v) for v in values]
    assert parsed == expected
    # lower without B
    values = [f"10{unit.lower()}" for unit in upper_units]
    parsed = [utils.parse_memory_size(v) for v in values]
    assert parsed == expected
    # upper blank without B
    values = [f"10 {unit}" for unit in upper_units]
    parsed = [utils.parse_memory_size(v) for v in values]
    assert parsed == expected
    # upper two blanks without B
    values = [f"10  {unit}" for unit in upper_units]
    parsed = [utils.parse_memory_size(v) for v in values]
    assert parsed == expected

    upper_units = ["B", "KB", "MB", "GB", "TB"]
    # upper with B
    values = [f"10{unit}" for unit in upper_units]
    parsed = [utils.parse_memory_size(v) for v in values]
    assert parsed == expected
    # lower with B
    values = [f"10{unit.lower()}" for unit in upper_units]
    parsed = [utils.parse_memory_size(v) for v in values]
    assert parsed == expected
    # upper blank with B
    values = [f"10 {unit}" for unit in upper_units]
    parsed = [utils.parse_memory_size(v) for v in values]
    assert parsed == expected
    # upper two blanks with B
    values = [f"10  {unit}" for unit in upper_units]
    parsed = [utils.parse_memory_size(v) for v in values]
    assert parsed == expected