def test_return_type_namedtuple(self):
result = []
for disk in self.disks:
info = self.disk_usage(disk.mountpoint)
result.append(
disks.DiskInfo(
mountpoint=disk.mountpoint,
free=info.free,
size=info.total
)
)
with nested(
patch.object(psutil, "disk_partitions", return_value=self.disks),
patch.object(psutil, "disk_usage", self.disk_usage)
):
self.assertEqual(disks.disks(), result)
def test_return_type_dict(self):
result = []
for disk in self.disks:
usage = self.disk_usage(disk.mountpoint)
result.append(
{
"mountpoint": disk.mountpoint,
"free": usage.free,
"size": usage.total
}
)
with nested(
patch.object(psutil, "disk_partitions", return_value=self.disks),
patch.object(psutil, "disk_usage", self.disk_usage)
):
self.assertEqual(disks.disks(as_dict=True), result)
def system_data(self, requery_timeoffset=False):
"""
Returns a dictionary of data containing information about the
agent. This is the information that is also passed along to
the master.
"""
# query the time offset and then cache it since
# this is typically a blocking operation
if config["agent_time_offset"] == "auto":
config["agent_time_offset"] = None
if requery_timeoffset or config["agent_time_offset"] is None:
ntplog.info(
"Querying ntp server %r for current time",
config["agent_ntp_server"])
ntp_client = NTPClient()
try:
pool_time = ntp_client.request(
config["agent_ntp_server"],
version=config["agent_ntp_server_version"])
except Exception as e:
ntplog.warning("Failed to determine network time: %s", e)
else:
config["agent_time_offset"] = \
int(pool_time.tx_time - time.time())
# format the offset for logging purposes
utcoffset = datetime.utcfromtimestamp(pool_time.tx_time)
iso_timestamp = utcoffset.isoformat()
ntplog.debug(
"network time: %s (local offset: %r)",
iso_timestamp, config["agent_time_offset"])
if config["agent_time_offset"] != 0:
ntplog.warning(
"Agent is %r second(s) off from ntp server at %r",
config["agent_time_offset"],
config["agent_ntp_server"])
data = {
"id": config["agent_id"],
"hostname": config["agent_hostname"],
"version": config.version,
"os_class": system.operating_system(),
"os_fullname": platform(),
"ram": int(config["agent_ram"]),
"cpus": config["agent_cpus"],
"cpu_name": cpu.cpu_name(),
"port": config["agent_api_port"],
"free_ram": memory.free_ram(),
"time_offset": config["agent_time_offset"] or 0,
"state": config["state"],
"mac_addresses": list(network.mac_addresses()),
"current_assignments": config.get(
"current_assignments", {}), # may not be set yet
"disks": disks.disks(as_dict=True)
}
try:
gpu_names = graphics.graphics_cards()
data["gpus"] = gpu_names
except graphics.GPULookupError:
pass
if "remote_ip" in config:
data.update(remote_ip=config["remote_ip"])
if config["farm_name"]:
data["farm_name"] = config["farm_name"]
return data
def reannounce(self, force=False):
"""
Method which is used to periodically contact the master. This
method is generally called as part of a scheduled task.
"""
# Attempt to acquire the reannounce lock but fail after 70%
# of the total time between reannouncements elapses. This should
# help prevent an accumulation of requests in the event the master
# is having issues.
try:
yield self.reannounce_lock.acquire(
config["agent_master_reannounce"] * .70
)
except utility.LockTimeoutError:
svclog.debug("Timed out while waiting to acquire reannounce_lock")
returnValue(None)
if not self.should_reannounce() and not force:
yield self.reannounce_lock.release()
returnValue(None)
svclog.debug("Announcing %s to master", config["agent_hostname"])
data = None
num_retry_errors = 0
while True: # for retries
try:
response = yield post_direct(
self.agent_api(),
data={
"state": config["state"],
"current_assignments": config.get(
"current_assignments", {} # may not be set yet
),
"free_ram": memory.free_ram(),
"disks": disks.disks(as_dict=True)
}
)
except (ResponseNeverReceived, RequestTransmissionFailed) as error:
num_retry_errors += 1
if num_retry_errors > config["broken_connection_max_retry"]:
svclog.error(
"Failed to announce self to the master, "
"caught try-again type errors %s times in a row.",
num_retry_errors)
break
else:
svclog.debug("While announcing self to master, caught "
"%s. Retrying immediately.",
error.__class__.__name__)
except Exception as error:
if force:
delay = http_retry_delay()
svclog.error(
"Failed to announce self to the master: %s. Will "
"retry in %s seconds.", error, delay)
deferred = Deferred()
reactor.callLater(delay, deferred.callback, None)
yield deferred
else:
# Don't retry because reannounce is called periodically
svclog.error(
"Failed to announce self to the master: %s. This "
"request will not be retried.", error)
break
else:
data = yield treq.json_content(response)
if response.code == OK:
config.master_contacted(announcement=True)
svclog.info("Announced self to the master server.")
break
elif response.code >= INTERNAL_SERVER_ERROR:
if not self.shutting_down:
delay = http_retry_delay()
svclog.warning(
"Could not announce self to the master server, "
"internal server error: %s. Retrying in %s "
"seconds.", data, delay)
deferred = Deferred()
reactor.callLater(delay, deferred.callback, None)
yield deferred
else:
svclog.warning(
"Could not announce to master. Not retrying "
"because of pending shutdown.")
break
elif response.code == NOT_FOUND:
svclog.warning("The master says it does not know about our "
"agent id. Posting as a new agent.")
yield self.post_agent_to_master()
break
# If this is a client problem retrying the request
# is unlikely to fix the issue so we stop here
elif response.code >= BAD_REQUEST:
svclog.error(
"Failed to announce self to the master, bad "
"request: %s. This request will not be retried.",
data)
break
else:
svclog.error(
"Unhandled error when posting self to the "
"master: %s (code: %s). This request will not be "
"retried.", data, response.code)
break
yield self.reannounce_lock.release()
returnValue(data)
def test_return_type(self):
self.assertIsInstance(disks.disks(), list)
