Exemple #1
0
    def cleanup_existing_pg(self, block: bool = False):
        """Clean up (remove) all existing placement groups.

        This scans through the placement_group_table to discover existing
        placement groups and calls remove_placement_group on all that
        match the ``_tune__`` prefix. This method is called at the beginning
        of the tuning run to clean up existing placement groups should the
        experiment be interrupted by a driver failure and resumed in the
        same driver script.

        Args:
            block (bool): If True, will wait until all placement groups are
                shut down.
        """
        should_cleanup = not int(
            os.getenv("TUNE_PLACEMENT_GROUP_CLEANUP_DISABLED", "0"))
        if should_cleanup:
            has_non_removed_pg_left = True
            while has_non_removed_pg_left:
                has_non_removed_pg_left = False
                for pid, info in placement_group_table().items():
                    if not info["name"].startswith(self._prefix):
                        continue
                    if info["state"] == "REMOVED":
                        continue
                    # If block=False, only run once
                    has_non_removed_pg_left = block
                    pg = get_placement_group(info["name"])
                    remove_placement_group(pg)
                time.sleep(0.1)
Exemple #2
0
import time
import ray
from ray.util.placement_group import (placement_group, placement_group_table,
                                      remove_placement_group)

if __name__ == "__main__":
    ray.init(num_cpus=2, resources={"extra_resources": 2})
    bundle_1 = {"CPU": 2}
    bundle_2 = {"extra_resources": 2}

    pg = placement_group([bundle_1, bundle_2], strategy="STRICT_PACK")

    # You can also use ray.wait.
    ready, unready = ray.wait([pg.ready()], timeout=5)
    print(f"placement group status:{ready}")
    print(placement_group_table(pg))

    time.sleep(10)