def test_not_enough_cpus(self):
self.data["job"]["cpus"] = int(total_cpus() * 10)
request = self.post(
data=self.data,
headers={"User-Agent": config["master_user_agent"]})
assign = self.instance_class()
result = assign.render(request)
self.assertEqual(result, NOT_DONE_YET)
self.assertTrue(request.finished)
self.assertEqual(request.responseCode, BAD_REQUEST)
self.assertEqual(len(request.written), 1)
response = loads(request.written[0])
self.assertEqual(response["requires_cpus"], int(total_cpus() * 10))
self.assertEqual(response["error"], "Not enough cpus")
def prepare_config(self):
for key in self._pop_config_keys:
config.pop(key, None)
config.update(
{
# Default to a uniform retry delay of one second
"agent_http_retry_delay_offset": 0.1,
"agent_http_retry_delay_factor": 0,
"shutting_down": False,
"jobtypes": {},
"current_assignments": {},
"agent_id": uuid.uuid4(),
"agent_http_persistent_connections": False,
"agent_shutdown_timeout": 3,
"master": PYFARM_AGENT_MASTER,
"agent_hostname": os.urandom(self.RAND_LENGTH).encode("hex"),
"agent_ram": memory.total_ram(),
"agent_cpus": cpu.total_cpus(),
"agent_api_port": randint(10000, 50000),
"free_ram": memory.free_ram(),
"agent_time_offset": randint(-50, 50),
"state": choice(AgentState),
"start": time.time(),
"agent_pretty_json": False,
"agent_html_template_reload": True,
"agent_master_reannounce": randint(5, 15),
}
)
def __init__(self):
minthreads = config["jobtype_logging_threadpool"]["min_threads"]
maxthreads = config["jobtype_logging_threadpool"]["max_threads"]
self.max_queued_lines = \
config["jobtype_logging_threadpool"]["max_queue_size"]
self.flush_lines = \
config["jobtype_logging_threadpool"]["flush_lines"]
self.stopped = False
if minthreads < 1:
raise ValueError(
"Config value "
"jobtype_logging_threadpool.min_threads must be >= 1")
# Calculate maxthreads if a value was not provided for us
if maxthreads == "auto":
auto_maxthreads = min(int(cpu.total_cpus() * 1.5), 20)
maxthreads = max(auto_maxthreads, minthreads)
if minthreads > maxthreads:
raise ValueError(
"Config value jobtype_logging_threadpool.min_threads cannot "
"be larger than jobtype_logging_threadpool.max_threads")
ThreadPool.__init__(
self,
minthreads=minthreads, maxthreads=maxthreads,
name=self.__class__.__name__)
def test_load(self):
# Make several attempts to test cpu.load(). Because this
# depends on the system load at the time we make a few
# attempts to get the results we expect.
for _ in range(15):
load = psutil.cpu_percent(.25) / cpu.total_cpus()
try:
self.assertApproximates(cpu.load(.25), load, .75)
except AssertionError:
continue
else:
break
else:
self.fail("Failed get a non-failing result after several attempts")
def _map_value(self, key, value):
"""
Some configuration values have keywords associated with
them, this function is responsible for returning the 'fixed'
value.
"""
if value == "auto":
if key == "agent_ram":
return memory.total_ram()
if key == "agent_cpus":
return cpu.total_cpus()
if key == "agent_hostname":
return network.hostname()
if key == "agent_static_root":
return abspath(
join(dirname(__file__), "http", "static"))
return value
def test_auto_max_maxthreads(self):
config["jobtype_logging_threadpool"]["max_threads"] = "auto"
pool = LoggerPool()
self.assertEqual(
pool.max, max(min(int(total_cpus() * 1.5), 20), pool.min))
def test_count(self):
self.assertEqual(psutil.cpu_count(), cpu.total_cpus())
def __init__(self):
self.args = None
self.parser = AgentArgumentParser(
usage="%(prog)s [status|start|stop]",
epilog="%(prog)s is a command line client for working with a "
"local agent. You can use it to stop, start, and report "
"the general status of a running agent process.")
# main subparser for start/stop/status/etc
subparsers = self.parser.add_subparsers(
help="individual operations %(prog)s can run")
start = subparsers.add_parser(
"start", help="starts the agent")
stop = subparsers.add_parser(
"stop", help="stops the agent")
status = subparsers.add_parser(
"status", help="query the 'running' state of the agent")
# relate a name and function to each subparser
start.set_defaults(target_name="start", target_func=self.start)
stop.set_defaults(target_name="stop", target_func=self.stop)
status.set_defaults(target_name="status", target_func=self.status)
# command line flags which configure the agent's network service
global_network = self.parser.add_argument_group(
"Agent Network Service",
description="Main flags which control the network services running "
"on the agent.")
global_network.add_argument(
"--port", config="agent_api_port", type=port,
type_kwargs=dict(get_uid=lambda: self.args.uid == 0),
help="The port number which the agent is either running on or "
"will run on when started. This port is also reported the "
"master when an agent starts. [default: %(default)s]")
global_network.add_argument(
"--host", config="agent_hostname",
help="The host to communicate with or hostname to present to the "
"master when starting. Defaults to the fully qualified "
"hostname.")
global_network.add_argument(
"--agent-api-username", default="agent", config=False,
help="The username required to access or manipulate the agent "
"using REST. [default: %(default)s]")
global_network.add_argument(
"--agent-api-password", default="agent", config=False,
help="The password required to access manipulate the agent "
"using REST. [default: %(default)s]")
global_network.add_argument(
"--agent-id", config="agent_id", type=uuid_type,
default=None,
help="The UUID used to identify this agent to the master. By "
"default the agent will attempt to load a cached value "
"however a specific UUID could be provided with this flag.")
global_network.add_argument(
"--agent-id-file", config="agent_id_file",
default=expanduser(expandvars(
config["agent_id_file_platform_defaults"][operating_system()])),
help="The location to store the agent's id. By default the path "
"is platform specific and defined by the "
"`agent_id_file_platform_defaults` key in the configuration. "
"[default: %(default)s]")
# command line flags for the connecting the master apis
global_apis = self.parser.add_argument_group(
"Network Resources",
description="Resources which the agent will be communicating with.")
global_apis.add_argument(
"--master", config="master",
help="This is a convenience flag which will allow you to set the "
"hostname for the master. By default this value will be "
"substituted in --master-api")
global_apis.add_argument(
"--master-api", config="master_api",
help="The location where the master's REST api is located. "
"[default: %(default)s]")
global_apis.add_argument(
"--master-api-version", config="master_api_version",
help="Sets the version of the master's REST api the agent should"
"use [default: %(default)s]")
# global command line flags which apply to top level
# process control
global_process = self.parser.add_argument_group(
"Process Control",
description="These settings apply to the parent process of the "
"agent and contribute to allowing the process to run "
"as other users or remain isolated in an environment. "
"They also assist in maintaining the 'running state' "
"via a process id file.")
global_process.add_argument(
"--pidfile", config="agent_lock_file",
help="The file to store the process id in. [default: %(default)s]")
global_process.add_argument(
"-n", "--no-daemon", default=False, action="store_true",
config=False,
help="If provided then do not run the process in the background.")
global_process.add_argument(
"--chdir", config="agent_chdir", type=isdir,
help="The working directory to change the agent into upon launch")
global_process.add_argument(
"--uid", type=uidgid, config=False,
type_kwargs=dict(get_id=getuid, check_id=getpwuid, set_id=setuid),
help="The user id to run the agent as. *This setting is "
"ignored on Windows.*")
global_process.add_argument(
"--gid", type=uidgid, config=False,
type_kwargs=dict(get_id=getgid, check_id=getgrgid, set_id=setgid),
help="The group id to run the agent as. *This setting is "
"ignored on Windows.*")
global_process.add_argument(
"--pdb-on-unhandled", action="store_true",
help="When set pdb.set_trace() will be called if an unhandled "
"error is caught in the logger")
# start general group
start_general_group = start.add_argument_group(
"General Configuration",
description="These flags configure parts of the agent related to "
"hardware, state, and certain timing and scheduling "
"attributes.")
start_general_group.add_argument(
"--state", default=AgentState.ONLINE, config=False,
type=enum, type_kwargs=dict(enum=AgentState),
help="The current agent state, valid values are "
"" + str(list(AgentState)) + ". [default: %(default)s]")
start_general_group.add_argument(
"--time-offset", config="agent_time_offset",
type=int, type_kwargs=dict(min_=0),
help="If provided then don't talk to the NTP server at all to "
"calculate the time offset. If you know for a fact that this "
"host's time is always up to date then setting this to 0 is "
"probably a safe bet.")
start_general_group.add_argument(
"--ntp-server", config="agent_ntp_server",
help="The default network time server this agent should query to "
"retrieve the real time. This will be used to help determine "
"the agent's clock skew if any. Setting this value to '' "
"will effectively disable this query. [default: %(default)s]")
start_general_group.add_argument(
"--ntp-server-version", config="agent_ntp_server_version",
type=int,
help="The version of the NTP server in case it's running an older"
"or newer version. [default: %(default)s]")
start_general_group.add_argument(
"--no-pretty-json", config="agent_pretty_json",
action="store_false",
help="If provided do not dump human readable json via the agent's "
"REST api")
start_general_group.add_argument(
"--shutdown-timeout", config="agent_shutdown_timeout",
type=int, type_kwargs=dict(min_=0),
help="How many seconds the agent should spend attempting to inform "
"the master that it's shutting down.")
start_general_group.add_argument(
"--updates-drop-dir", config="agent_updates_dir",
help="The directory to drop downloaded updates in. This should be "
"the same directory pyfarm-supervisor will look for updates in. "
"[default: %(default)s]")
start_general_group.add_argument(
"--run-control-file", config="run_control_file",
default=expanduser(expandvars(
config["run_control_file_by_platform"][operating_system()])),
help="The path to a file that will signal to the supervisor that "
"agent is supposed to be restarted if it stops for whatever "
"reason."
"[default: %(default)s]")
start_general_group.add_argument(
"--farm-name", config="farm_name",
default=None,
help="The name of the farm the agent should join. If unset, the "
"agent will join any farm.")
# start hardware group
start_hardware_group = start.add_argument_group(
"Physical Hardware",
description="Command line flags which describe the hardware of "
"the agent.")
start_hardware_group.add_argument(
"--cpus", default=cpu.total_cpus(),
config="agent_cpus", type=int,
help="The total amount of cpus installed on the "
"system. Defaults to the number of cpus installed "
"on the system.")
start_hardware_group.add_argument(
"--ram", default=memory.total_ram(),
config="agent_ram", type=int,
help="The total amount of ram installed on the system in "
"megabytes. Defaults to the amount of ram the "
"system has installed.")
# start interval controls
start_interval_group = start.add_argument_group(
"Interval Controls",
description="Controls which dictate when certain internal "
"intervals should occur.")
start_interval_group.add_argument(
"--ram-check-interval",
config="agent_ram_check_interval", type=int,
help="How often ram resources should be checked for changes. "
"The amount of memory currently being consumed on the system "
"is checked after certain events occur such as a process but "
"this flag specifically controls how often we should check "
"when no such events are occurring. [default: %(default)s]")
start_interval_group.add_argument(
"--ram-max-report-frequency",
config="agent_ram_max_report_frequency", type=int,
help="This is a limiter that prevents the agent from reporting "
"memory changes to the master more often than a specific "
"time interval. This is done in order to ensure that when "
"100s of events fire in a short period of time cause changes "
"in ram usage only one or two will be reported to the "
"master. [default: %(default)s]")
start_interval_group.add_argument(
"--ram-report-delta", config="agent_ram_report_delta", type=int,
help="Only report a change in ram if the value has changed "
"at least this many megabytes. [default: %(default)s]")
start_interval_group.add_argument(
"--master-reannounce", config="agent_master_reannounce", type=int,
help="Controls how often the agent should reannounce itself "
"to the master. The agent may be in contact with the master "
"more often than this however during long period of "
"inactivity this is how often the agent will 'inform' the "
"master the agent is still online.")
# start logging options
logging_group = start.add_argument_group(
"Logging Options",
description="Settings which control logging of the agent's parent "
"process and/or any subprocess it runs.")
logging_group.add_argument(
"--log", config="agent_log",
help="If provided log all output from the agent to this path. "
"This will append to any existing log data. [default: "
"%(default)s]")
logging_group.add_argument(
"--capture-process-output", config="jobtype_capture_process_output",
action="store_true",
help="If provided then all log output from each process launched "
"by the agent will be sent through agent's loggers.")
logging_group.add_argument(
"--task-log-dir", config="jobtype_task_logs",
type=isdir, type_kwargs=dict(create=True),
help="The directory tasks should log to.")
# network options for the agent when start is called
start_network = start.add_argument_group(
"Network Service",
description="Controls how the agent is seen or interacted with "
"by external services such as the master.")
start_network.add_argument(
"--ip-remote", type=ip, config=False,
help="The remote IPv4 address to report. In situation where the "
"agent is behind a firewall this value will typically be "
"different.")
start_manhole = start.add_argument_group(
"Manhole Service",
description="Controls the manhole service which allows a telnet "
"connection to be made directly into the agent as "
"it's running.")
start_manhole.add_argument(
"--enable-manhole", config="agent_manhole",
action="store_true",
help="When provided the manhole service will be started once the "
"reactor is running.")
start_manhole.add_argument(
"--manhole-port", config="agent_manhole_port", type=port,
type_kwargs=dict(get_uid=lambda: self.args.uid == 0),
help="The port the manhole service should run on if enabled.")
start_manhole.add_argument(
"--manhole-username", config="agent_manhole_username",
help="The telnet username that's allowed to connect to the "
"manhole service running on the agent.")
start_manhole.add_argument(
"--manhole-password", config="agent_manhole_password",
help="The telnet password to use when connecting to the "
"manhole service running on the agent.")
# various options for how the agent will interact with the
# master server
start_http_group = start.add_argument_group(
"HTTP Configuration",
description="Options for how the agent will interact with the "
"master's REST api and how it should run it's own "
"REST api.")
start_http_group.add_argument(
"--html-templates-reload", config="agent_html_template_reload",
action="store_true",
help="If provided then force Jinja2, the html template system, "
"to check the file system for changes with every request. "
"This flag should not be used in production but is useful "
"for development and debugging purposes.")
start_http_group.add_argument(
"--static-files", config="agent_static_root", type=isdir,
help="The default location where the agent's http server should "
"find static files to serve.")
start_http_group.add_argument(
"--http-retry-delay-offset",
config="agent_http_retry_delay_offset", type=number,
help="If a http request to the master has failed, wait at least "
"this amount of time before resending the request.")
start_http_group.add_argument(
"--http-retry-delay-factor",
config="agent_http_retry_delay_factor", type=number,
help="The value provided here is used in combination with "
"--http-retry-delay-offset to calculate the retry delay. "
"This is used as a multiplier against random() before being "
"added to the offset.")
jobtype_group = start.add_argument_group("Job Types")
jobtype_group.add_argument(
"--jobtype-no-cache", config="jobtype_enable_cache",
action="store_true",
help="If provided then do not cache job types, always directly "
"retrieve them. This is beneficial if you're testing the "
"agent or a new job type class.")
# options when stopping the agent
stop_group = stop.add_argument_group(
"optional flags",
description="Flags that control how the agent is stopped")
stop_group.add_argument(
"--no-wait", default=False, action="store_true", config=False,
help="If provided then don't wait on the agent to shut itself "
"down. By default we would want to wait on each task to stop "
"so we can catch any errors and then finally wait on the "
"agent to shutdown too. If you're in a hurry or stopping a "
"bunch of agents at once then setting this flag will let the "
"agent continue to stop itself without waiting for each agent")
