def do_draw(self, data: Any) -> ht.VMSize:
import hypothesis.internal.conjecture.utils as d
idx = d.integer_range(data, 0, 1_000_000_000)
r = random.Random(idx)
def draw_value() -> Optional[Any]:
rtype_draw = r.randint(0, 4)
if rtype_draw == 0:
return r.randint(0, 100)
elif rtype_draw == 1:
return r.random() * 100
elif rtype_draw == 2:
def draw_letter():
return r.choice(string.ascii_letters)
return "".join([draw_letter() for n in range(r.randint(0, 100))])
elif rtype_draw == 3:
return r.random() < 0.5
else:
return None
hostname = "n-o-d-e_-{}".format(r.randint(1, 1000000))
resources: Dict[str, Optional[Any]] = {}
num_resources = r.randint(0, 10)
for n in range(num_resources):
rname = "res-{}".format(n)
resources[rname] = draw_value()
node = SchedulerNode(ht.Hostname(hostname), resources, "bucket-id-123")
for job_id in range(r.randint(0, 10)):
node.assign(str(job_id))
num_meta = r.randint(0, 10)
for n in range(num_meta):
mname = "meta-{}".format(n)
node.metadata[mname] = draw_value()
for rname, rvalue in node.resources.items():
if r.random() > 0.5:
if isinstance(rvalue, int):
node.available[rname] = rvalue - r.randint(0, rvalue + 1)
elif isinstance(rvalue, float):
node.available[rname] = rvalue * r.random()
elif isinstance(rvalue, bool):
node.available[rname] = rvalue ^ (r.random() < 0.5)
elif rvalue is None:
# in theory you can change a null resource
# into an actual value as available
if r.random() < 0.25:
node.available[rname] = draw_value()
return node
def from_dict(d: typing.Dict) -> "SchedulerNode":
hostname = ht.Hostname(d["hostname"])
resources = d.get("resources", {})
available = d.get("available", {})
metadata = d.get("metadata", {})
job_ids = d.get("job_ids", [])
bucket_id = d.get("bucket-id")
ret = SchedulerNode(hostname, resources, bucket_id)
for job_id in job_ids:
ret.assign(job_id)
ret.available.update(available)
ret.metadata.update(metadata)
return ret
def __init__(
self,
hostname: str,
resources: typing.Optional[dict] = None,
bucket_id: typing.Optional[ht.BucketId] = None,
) -> None:
resources = resources or ht.ResourceDict({})
private_ip: typing.Optional[ht.IpAddress]
if SchedulerNode.ignore_hostnames:
private_ip = None
else:
try:
private_ip = ht.IpAddress(socket.gethostbyname(hostname))
except Exception as e:
logging.warning("Could not find private ip for %s: %s", hostname, e)
private_ip = None
Node.__init__(
self,
node_id=DelayedNodeId(ht.NodeName(hostname)),
name=ht.NodeName(hostname),
nodearray=ht.NodeArrayName("unknown"),
bucket_id=bucket_id or ht.BucketId(str(uuid4())),
hostname=ht.Hostname(hostname),
private_ip=private_ip,
instance_id=None,
vm_size=ht.VMSize("unknown"),
location=ht.Location("unknown"),
spot=False,
vcpu_count=1,
memory=ht.Memory(0, "b"),
infiniband=False,
state=ht.NodeStatus("running"),
target_state=ht.NodeStatus("running"),
power_state=ht.NodeStatus("running"),
exists=True,
placement_group=None,
managed=False,
resources=ht.ResourceDict(resources),
software_configuration=ImmutableOrderedDict({}),
keep_alive=False,
)
def __init__(
self,
hostname: str,
resources: Optional[dict] = None,
vm_size: Optional[ht.VMSize] = None,
location: Optional[ht.Location] = None,
vcpu_count: Optional[int] = None,
memory: Optional[ht.Memory] = None,
placement_group: Optional[ht.PlacementGroup] = None,
) -> None:
resources = resources or ht.ResourceDict({})
if vm_size:
assert (vm_size and location
), "You must specify location when specifying vm_size"
vm_size = vm_size or ht.VMSize("unknown")
location = location or ht.Location("unknown")
aux = vm_sizes.get_aux_vm_size_info(location, vm_size)
Node.__init__(
self,
node_id=DelayedNodeId(ht.NodeName(hostname)),
name=ht.NodeName(hostname),
nodearray=ht.NodeArrayName("unknown"),
bucket_id=ht.BucketId("unknown"),
hostname=ht.Hostname(hostname),
private_ip=None,
vm_size=vm_size,
location=location,
spot=False,
vcpu_count=aux.vcpu_count,
memory=aux.memory,
infiniband=False,
state=ht.NodeStatus("running"),
power_state=ht.NodeStatus("running"),
exists=True,
placement_group=placement_group,
managed=False,
resources=ht.ResourceDict(resources),
software_configuration=ImmutableOrderedDict({}),
keep_alive=True,
)
assert self.exists
def node_from_bucket(
bucket: "NodeBucket",
new_node_name: ht.NodeName,
state: ht.NodeStatus,
target_state: ht.NodeStatus,
power_state: ht.NodeStatus,
hostname: Optional[ht.Hostname] = None,
placement_group: Optional[ht.PlacementGroup] = None,
exists: bool = True,
) -> "Node":
if hostname is None:
hostname = ht.Hostname(util.uuid("hostname"))
from hpc.autoscale.node.node import Node
return Node(
node_id=DelayedNodeId(new_node_name),
name=new_node_name,
nodearray=bucket.nodearray,
bucket_id=bucket.bucket_id,
vm_size=bucket.vm_size,
hostname=hostname,
private_ip=None,
instance_id=None,
location=bucket.location,
spot=bucket.spot,
vcpu_count=bucket.vcpu_count,
memory=bucket.memory,
infiniband=False, # TODO
state=state,
target_state=target_state,
power_state=power_state,
exists=exists,
placement_group=placement_group,
managed=True,
resources=ht.ResourceDict(bucket.resources),
software_configuration=bucket.software_configuration,
keep_alive=False,
)
def __init__(
self,
hostname: str,
resources: typing.Optional[dict] = None,
bucket_id: typing.Optional[ht.BucketId] = None,
**overrides: typing.Any,
) -> None:
resources = resources or ht.ResourceDict({})
node_init_args = dict(
self=self,
node_id=DelayedNodeId(ht.NodeName(hostname)),
name=ht.NodeName(hostname),
nodearray=ht.NodeArrayName("unknown"),
bucket_id=bucket_id or ht.BucketId(str(uuid4())),
hostname=ht.Hostname(hostname),
private_ip=None,
instance_id=None,
vm_size=ht.VMSize("unknown"),
location=ht.Location("unknown"),
spot=False,
vcpu_count=1,
memory=ht.Memory(0, "b"),
infiniband=False,
state=ht.NodeStatus("running"),
target_state=ht.NodeStatus("running"),
power_state=ht.NodeStatus("running"),
exists=True,
placement_group=None,
managed=False,
resources=ht.ResourceDict(resources),
software_configuration=ImmutableOrderedDict({}),
keep_alive=False,
)
node_init_args.update(overrides)
Node.__init__(**node_init_args)
def hostname_required(self) -> ht.Hostname:
if self.hostname is None:
raise AssertionError("null hostname")
return ht.Hostname(str(self.hostname))
def hostname_or_uuid(self) -> Optional[ht.Hostname]:
return ht.Hostname(self.__hostname
or self.delayed_node_id.transient_id)