def post(self):
"""
A ``POST`` to this endpoint will either create or update an existing
agent. The ``port`` and ``id`` columns will determine if an
agent already exists.
* If an agent is found matching the ``port`` and ``id``
columns from the request the existing model will be updated and
the resulting data and the ``OK`` code will be returned.
* If we don't find an agent matching the ``port`` and ``id``
however a new agent will be created and the resulting data and the
``CREATED`` code will be returned.
.. note::
The ``remote_ip`` field is not required and should typically
not be included in a request. When not provided ``remote_ip``
is be populated by the server based off of the ip of the
incoming request. Providing ``remote_ip`` in your request
however will override this behavior.
.. http:post:: /api/v1/agents/ HTTP/1.1
**Request**
.. sourcecode:: http
POST /api/v1/agents/ HTTP/1.1
Accept: application/json
{
"cpu_allocation": 1.0,
"cpus": 14,
"free_ram": 133,
"hostname": "agent1",
"id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
"remote_ip": "10.196.200.115",
"port": 64994,
"ram": 2157,
"ram_allocation": 0.8,
"state": 8
}
**Response (agent created)**
.. sourcecode:: http
HTTP/1.1 201 CREATED
Content-Type: application/json
{
"cpu_allocation": 1.0,
"cpus": 14,
"use_address": "remote",
"free_ram": 133,
"time_offset": 0,
"hostname": "agent1",
"id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
"port": 64994,
"ram": 2157,
"ram_allocation": 0.8,
"state": "online",
"remote_ip": "10.196.200.115"
}
**Response (existing agent updated)**
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"cpu_allocation": 1.0,
"cpus": 14,
"use_address": "remote",
"free_ram": 133,
"time_offset": 0,
"hostname": "agent1",
"id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
"port": 64994,
"ram": 2157,
"ram_allocation": 0.8,
"state": "online",
"remote_ip": "10.196.200.115"
}
:statuscode 201:
a new agent was created
:statuscode 200:
an existing agent is updated with data from the request
:statuscode 400:
there was something wrong with the request (such as
invalid columns being included)
"""
# Read in and convert the id field
try:
g.json["id"] = uuid.UUID(g.json["id"])
except KeyError:
return jsonify(error="`id` not provided"), BAD_REQUEST
# If the remote user agent is not "PyFarm/1.0 (agent)", this is not an
# announce by the agent itself. It could be some other client editing
# the agent.
if request.headers.get("User-Agent", "") == "PyFarm/1.0 (agent)":
# Set remote_ip if it did not come in with the request
g.json.setdefault("remote_ip", request.remote_addr)
farm_name = g.json.pop("farm_name", None)
if farm_name and farm_name != OUR_FARM_NAME:
return jsonify(error="Wrong farm name"), BAD_REQUEST
current_assignments = g.json.pop("current_assignments", None)
mac_addresses = g.json.pop("mac_addresses", None)
# TODO return BAD_REQUEST on bad mac addresses
if mac_addresses is not None:
mac_addresses = [x.lower() for x in mac_addresses if MAC_RE.match(x)]
gpus = g.json.pop("gpus", None)
disks = g.json.pop("disks", None)
state = g.json.pop("state", None)
agent = Agent.query.filter_by(
port=g.json["port"], id=g.json["id"]).first()
if agent is None:
try:
agent = Agent(**g.json)
# There may be something wrong with one of the fields
# that's causing our sqlalchemy model raise a ValueError.
except ValueError as e:
return jsonify(error=str(e)), BAD_REQUEST
default_versions = SoftwareVersion.query.filter_by(default=True)
for version in default_versions:
agent.software_versions.append(version)
if mac_addresses is not None:
for address in mac_addresses:
mac_address = AgentMacAddress(agent=agent,
mac_address=address)
db.session.add(mac_address)
if gpus is not None:
for gpu_name in gpus:
gpu = GPU.query.filter_by(
fullname=gpu_name).first()
if not gpu:
gpu = GPU(fullname=gpu_name)
db.session.add(gpu)
agent.gpus.append(gpu)
if disks is not None:
for disk_dict in disks:
disk = AgentDisk(agent=agent,
mountpoint=disk_dict["mountpoint"],
size=disk_dict["size"],
free=disk_dict["free"])
db.session.add(disk)
if state is not None:
agent.state = state
db.session.add(agent)
try:
db.session.commit()
except Exception as e:
e = e.args[0].lower()
error = "Unhandled error: %s. This is often an issue " \
"with the agent's data for `ip`, `hostname` and/or " \
"`port` not being unique enough. In other cases " \
"this can sometimes happen if the underlying " \
"database driver is either non-compliant with " \
"expectations or we've encountered a database error " \
"that we don't know how to handle yet. If the " \
"latter is the case, please report this as a bug." % e
return jsonify(error=error), INTERNAL_SERVER_ERROR
else:
agent_data = agent.to_dict(unpack_relationships=["tags"])
logger.info("Created agent %r: %r", agent.id, agent_data)
assign_tasks.delay()
return jsonify(agent_data), CREATED
else:
updated = False
for key in g.json.copy():
value = g.json.pop(key)
if not hasattr(agent, key):
return jsonify(
error="Agent has no such column `%s`" % key), \
BAD_REQUEST
if getattr(agent, key) != value:
try:
setattr(agent, key, value)
except Exception as e:
return jsonify(
error="Error while setting `%s`: %s" % (key, e)), \
BAD_REQUEST
else:
updated = True
if mac_addresses is not None:
updated = True
for existing_address in agent.mac_addresses:
if existing_address.mac_address.lower() not in mac_addresses:
logger.debug("Existing address %s is not in supplied "
"mac addresses, for agent %s, removing it.",
existing_address.mac_address,
agent.hostname)
agent.mac_addresses.remove(existing_address)
else:
mac_addresses.remove(
existing_address.mac_address.lower())
for new_address in mac_addresses:
mac_address = AgentMacAddress(
agent=agent, mac_address=new_address)
db.session.add(mac_address)
if gpus is not None:
updated = True
for existing_gpu in agent.gpus:
if existing_gpu.fullname not in gpus:
logger.debug("Existing gpu %s is not in supplied "
"gpus, for agent %s, removing it.",
existing_address.mac_address,
agent.hostname)
agent.gpus.remove(existing_gpu)
else:
gpus.remove(existing_gpu.fullname)
for gpu_name in gpus:
gpu = GPU.query.filter_by(fullname=gpu_name).first()
if not gpu:
gpu = GPU(fullname=gpu_name)
db.session.add(gpu)
agent.gpus.append(gpu)
if disks is not None:
for old_disk in agent.disks:
db.session.delete(old_disk)
for disk_dict in disks:
disk = AgentDisk(agent=agent,
mountpoint=disk_dict["mountpoint"],
size=disk_dict["size"],
free=disk_dict["free"])
db.session.add(disk)
if state is not None and agent.state != _AgentState.DISABLED:
agent.state = state
# TODO Only do that if this is really the agent speaking to us.
failed_tasks = []
if (current_assignments is not None and
agent.state != AgentState.OFFLINE):
fail_missing_assignments(agent, current_assignments)
if updated or failed_tasks:
agent.last_heard_from = datetime.utcnow()
db.session.add(agent)
try:
db.session.commit()
except Exception as e:
return jsonify(error="Unhandled error: %s" % e), \
INTERNAL_SERVER_ERROR
else:
agent_data = agent.to_dict(unpack_relationships=["tags"])
logger.info("Updated agent %r: %r", agent.id, agent_data)
for task in failed_tasks:
task.job.update_state()
db.session.commit()
assign_tasks.delay()
return jsonify(agent_data), OK
def post(self):
"""
A ``POST`` to this endpoint will either create or update an existing
agent. The ``port`` and ``id`` columns will determine if an
agent already exists.
* If an agent is found matching the ``port`` and ``id``
columns from the request the existing model will be updated and
the resulting data and the ``OK`` code will be returned.
* If we don't find an agent matching the ``port`` and ``id``
however a new agent will be created and the resulting data and the
``CREATED`` code will be returned.
.. note::
The ``remote_ip`` field is not required and should typically
not be included in a request. When not provided ``remote_ip``
is be populated by the server based off of the ip of the
incoming request. Providing ``remote_ip`` in your request
however will override this behavior.
.. http:post:: /api/v1/agents/ HTTP/1.1
**Request**
.. sourcecode:: http
POST /api/v1/agents/ HTTP/1.1
Accept: application/json
{
"cpu_allocation": 1.0,
"cpus": 14,
"free_ram": 133,
"hostname": "agent1",
"id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
"remote_ip": "10.196.200.115",
"port": 64994,
"ram": 2157,
"ram_allocation": 0.8,
"state": 8
}
**Response (agent created)**
.. sourcecode:: http
HTTP/1.1 201 CREATED
Content-Type: application/json
{
"cpu_allocation": 1.0,
"cpus": 14,
"use_address": "remote",
"free_ram": 133,
"time_offset": 0,
"hostname": "agent1",
"id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
"port": 64994,
"ram": 2157,
"ram_allocation": 0.8,
"state": "online",
"remote_ip": "10.196.200.115"
}
**Response (existing agent updated)**
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"cpu_allocation": 1.0,
"cpus": 14,
"use_address": "remote",
"free_ram": 133,
"time_offset": 0,
"hostname": "agent1",
"id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
"port": 64994,
"ram": 2157,
"ram_allocation": 0.8,
"state": "online",
"remote_ip": "10.196.200.115"
}
:statuscode 201:
a new agent was created
:statuscode 200:
an existing agent is updated with data from the request
:statuscode 400:
there was something wrong with the request (such as
invalid columns being included)
"""
# Read in and convert the id field
try:
g.json["id"] = uuid.UUID(g.json["id"])
except KeyError:
return jsonify(error="`id` not provided"), BAD_REQUEST
# If the remote user agent is not "PyFarm/1.0 (agent)", this is not an
# announce by the agent itself. It could be some other client editing
# the agent.
if request.headers.get("User-Agent", "") == "PyFarm/1.0 (agent)":
# Set remote_ip if it did not come in with the request
g.json.setdefault("remote_ip", request.remote_addr)
farm_name = g.json.pop("farm_name", None)
if farm_name and farm_name != OUR_FARM_NAME:
return jsonify(error="Wrong farm name"), BAD_REQUEST
current_assignments = g.json.pop("current_assignments", None)
mac_addresses = g.json.pop("mac_addresses", None)
# TODO return BAD_REQUEST on bad mac addresses
if mac_addresses is not None:
mac_addresses = [
x.lower() for x in mac_addresses if MAC_RE.match(x)
]
gpus = g.json.pop("gpus", None)
disks = g.json.pop("disks", None)
state = g.json.pop("state", None)
agent = Agent.query.filter_by(port=g.json["port"],
id=g.json["id"]).first()
if agent is None:
try:
agent = Agent(**g.json)
# There may be something wrong with one of the fields
# that's causing our sqlalchemy model raise a ValueError.
except ValueError as e:
return jsonify(error=str(e)), BAD_REQUEST
default_versions = SoftwareVersion.query.filter_by(default=True)
for version in default_versions:
agent.software_versions.append(version)
if mac_addresses is not None:
for address in mac_addresses:
mac_address = AgentMacAddress(agent=agent,
mac_address=address)
db.session.add(mac_address)
if gpus is not None:
for gpu_name in gpus:
gpu = GPU.query.filter_by(fullname=gpu_name).first()
if not gpu:
gpu = GPU(fullname=gpu_name)
db.session.add(gpu)
agent.gpus.append(gpu)
if disks is not None:
for disk_dict in disks:
disk = AgentDisk(agent=agent,
mountpoint=disk_dict["mountpoint"],
size=disk_dict["size"],
free=disk_dict["free"])
db.session.add(disk)
if state is not None:
agent.state = state
db.session.add(agent)
try:
db.session.commit()
except Exception as e:
e = e.args[0].lower()
error = "Unhandled error: %s. This is often an issue " \
"with the agent's data for `ip`, `hostname` and/or " \
"`port` not being unique enough. In other cases " \
"this can sometimes happen if the underlying " \
"database driver is either non-compliant with " \
"expectations or we've encountered a database error " \
"that we don't know how to handle yet. If the " \
"latter is the case, please report this as a bug." % e
return jsonify(error=error), INTERNAL_SERVER_ERROR
else:
agent_data = agent.to_dict(unpack_relationships=["tags"])
logger.info("Created agent %r: %r", agent.id, agent_data)
assign_tasks.delay()
return jsonify(agent_data), CREATED
else:
updated = False
for key in g.json.copy():
value = g.json.pop(key)
if not hasattr(agent, key):
return jsonify(
error="Agent has no such column `%s`" % key), \
BAD_REQUEST
if getattr(agent, key) != value:
try:
setattr(agent, key, value)
except Exception as e:
return jsonify(
error="Error while setting `%s`: %s" % (key, e)), \
BAD_REQUEST
else:
updated = True
if mac_addresses is not None:
updated = True
for existing_address in agent.mac_addresses:
if existing_address.mac_address.lower(
) not in mac_addresses:
logger.debug(
"Existing address %s is not in supplied "
"mac addresses, for agent %s, removing it.",
existing_address.mac_address, agent.hostname)
agent.mac_addresses.remove(existing_address)
else:
mac_addresses.remove(
existing_address.mac_address.lower())
for new_address in mac_addresses:
mac_address = AgentMacAddress(agent=agent,
mac_address=new_address)
db.session.add(mac_address)
if gpus is not None:
updated = True
for existing_gpu in agent.gpus:
if existing_gpu.fullname not in gpus:
logger.debug(
"Existing gpu %s is not in supplied "
"gpus, for agent %s, removing it.",
existing_address.mac_address, agent.hostname)
agent.gpus.remove(existing_gpu)
else:
gpus.remove(existing_gpu.fullname)
for gpu_name in gpus:
gpu = GPU.query.filter_by(fullname=gpu_name).first()
if not gpu:
gpu = GPU(fullname=gpu_name)
db.session.add(gpu)
agent.gpus.append(gpu)
if disks is not None:
for old_disk in agent.disks:
db.session.delete(old_disk)
for disk_dict in disks:
disk = AgentDisk(agent=agent,
mountpoint=disk_dict["mountpoint"],
size=disk_dict["size"],
free=disk_dict["free"])
db.session.add(disk)
if state is not None and agent.state != _AgentState.DISABLED:
agent.state = state
# TODO Only do that if this is really the agent speaking to us.
failed_tasks = []
if (current_assignments is not None
and agent.state != AgentState.OFFLINE):
fail_missing_assignments(agent, current_assignments)
if updated or failed_tasks:
agent.last_heard_from = datetime.utcnow()
db.session.add(agent)
try:
db.session.commit()
except Exception as e:
return jsonify(error="Unhandled error: %s" % e), \
INTERNAL_SERVER_ERROR
else:
agent_data = agent.to_dict(unpack_relationships=["tags"])
logger.info("Updated agent %r: %r", agent.id, agent_data)
for task in failed_tasks:
task.job.update_state()
db.session.commit()
assign_tasks.delay()
return jsonify(agent_data), OK