def test_unique(self):
job = Job()
tagA = Tag()
tagB = Tag()
tagA.jobs = [job]
tagA.tag = "foo0"
tagB.jobs = [job]
tagB.tag = "foo1"
db.session.add_all([job, tagA, tagB])
with self.assertRaises(DatabaseError):
db.session.commit()
def test_null(self):
with self.assertRaises(DatabaseError):
model = Tag()
db.session.add(model)
db.session.commit()
db.session.remove()
with self.assertRaises(DatabaseError):
tag = Tag()
tag.tag = "foo789"
db.session.add(model)
db.session.commit()
def test_insert(self):
# A job can not be created without a jobtype, create one first
jobtype = JobType()
jobtype.name = "foo"
jobtype.description = "this is a job type"
jobtype.classname = "Foobar"
jobtype.code = dedent("""
class Foobar(JobType):
pass""").encode("utf-8")
jobtype.mode = JobTypeLoadMode.OPEN
db.session.add(jobtype)
job = Job()
job.job_type = jobtype
tag = Tag()
tag.jobs = [job]
tag.tag = "foo456"
db.session.add_all([tag, job])
db.session.commit()
model_id = tag.id
job_id = job.id
db.session.remove()
result = Tag.query.filter_by(id=model_id).first()
self.assertEqual(result.tag, "foo456")
self.assertEqual(result.jobs[0].id, job_id)
def add_tag_on_jobs():
job_ids = request.form.getlist("job_id")
tag_name = request.form["tag"].strip()
tag = Tag.query.filter_by(tag=tag_name).first()
if not tag:
tag = Tag(tag=tag_name)
db.session.add(tag)
for job_id in job_ids:
job = Job.query.filter_by(id=job_id).first()
if not job:
return (render_template("pyfarm/error.html",
error="Job %s not found" % job_id),
NOT_FOUND)
if tag not in job.tags:
job.tags.append(tag)
db.session.add(job)
db.session.commit()
flash("Tag %s has been added to selected jobs." % tag_name)
if "next" in request.args:
return redirect(request.args.get("next"), SEE_OTHER)
else:
return redirect(url_for("jobs_index_ui"), SEE_OTHER)
def test_insert(self):
# A job can not be created without a jobtype, create one first
jobtype = JobType()
jobtype.name = "foo"
jobtype.description = "this is a job type"
jobtype_version = JobTypeVersion()
jobtype_version.jobtype = jobtype
jobtype_version.version = 1
jobtype_version.classname = "Foobar"
jobtype_version.code = ("""
class Foobar(JobType):
pass""").encode("utf-8")
db.session.add(jobtype_version)
queue = JobQueue()
queue.name = "FooQueue"
job = Job()
job.title = "Test Job"
job.jobtype_version = jobtype_version
job.queue = queue
tag = Tag()
tag.jobs = [job]
tag.tag = "foo456"
db.session.add_all([tag, job])
db.session.commit()
model_id = tag.id
job_id = job.id
db.session.remove()
result = Tag.query.filter_by(id=model_id).first()
self.assertEqual(result.tag, "foo456")
self.assertEqual(result.jobs[0].id, job_id)
def update_tag_requirements_in_job(job_id):
job = Job.query.filter_by(id=job_id).first()
if not job:
return (render_template("pyfarm/error.html",
error="Job %s not found" % job_id), NOT_FOUND)
tagnames = request.form["tag_requirements"].split(" ")
tagnames = [x.strip() for x in tagnames if not x == ""]
job.tag_requirements = []
for name in tagnames:
negate = False
if name.startswith("-"):
if len(name) < 2:
return (render_template("pyfarm/error.html",
error="Tag too short: %s" % name),
NOT_FOUND)
negate = True
name = name[1:]
tag = Tag.query.filter_by(tag=name).first()
if not tag:
tag = Tag(tag=name)
db.session.add(tag)
tag_requirement = JobTagRequirement(job=job, tag=tag, negate=negate)
db.session.add(tag_requirement)
db.session.commit()
flash("Tag Requirements for job %s have been updated." % job.title)
return redirect(url_for("single_job_ui", job_id=job.id), SEE_OTHER)
def update_tags_in_agent(agent_id):
agent = Agent.query.filter_by(id=agent_id).first()
if not agent:
return (render_template(
"pyfarm/error.html", error="Agent %s not found" % agent_id),
NOT_FOUND)
tagnames = request.form["tags"].split(" ")
tagnames = [x.strip() for x in tagnames if not x == ""]
tags = []
for name in tagnames:
tag = Tag.query.filter_by(tag=name).first()
if not tag:
tag = Tag(tag=name)
db.session.add(tag)
tags.append(tag)
agent.tags = tags
db.session.add(agent)
db.session.commit()
flash("Tags for agent %s have been updated." % agent.hostname)
return redirect(url_for("single_agent_ui", agent_id=agent.id), SEE_OTHER)
def test_tags_validation(self):
for agent_foobar in self.models(limit=1):
tag = Tag()
tag.agent = [agent_foobar]
tag.tag = "foo123"
db.session.add(tag)
db.session.commit()
self.assertEqual(tag.tag, "foo123")
def test_tags_validation_error(self):
for agent_foobar in self.models(limit=1):
tag = Tag()
tag.agents = [agent_foobar]
tag.tag = None
with self.assertRaises(DatabaseError):
db.session.add(tag)
db.session.commit()
def add_tag_requirement_on_jobs():
job_ids = request.form.getlist("job_id")
tag_name = request.form["tag"].strip()
negate = False
if tag_name[0] == "-":
if len(tag_name) < 2:
return (render_template(
"pyfarm/error.html",
error="Tag must be at least one character long"), NOT_FOUND)
else:
negate = True
tag_name = tag_name[1:]
tag = Tag.query.filter_by(tag=tag_name).first()
if not tag:
tag = Tag(tag=tag_name)
db.session.add(tag)
for job_id in job_ids:
job = Job.query.filter_by(id=job_id).first()
if not job:
return (render_template("pyfarm/error.html",
error="Job %s not found" % job_id),
NOT_FOUND)
tag_requirement = JobTagRequirement.query.filter_by(job=job,
tag=tag).first()
if not tag_requirement:
tag_requirement = JobTagRequirement(job=job,
tag=tag,
negate=negate)
db.session.add(tag_requirement)
elif tag_requirement.negate != negate:
tag_requirement.negate = negate
db.session.add(tag_requirement)
db.session.commit()
flash("Tag requirement %s has been added to selected jobs." % tag_name)
if "next" in request.args:
return redirect(request.args.get("next"), SEE_OTHER)
else:
return redirect(url_for("jobs_index_ui"), SEE_OTHER)
def test_tags(self):
for agent_foobar in self.models(limit=1):
db.session.add(agent_foobar)
tags = []
rand = lambda: uuid.uuid4().hex
for tag_name in (rand(), rand(), rand()):
tag = Tag()
tag.tag = tag_name
tags.append(tag_name)
agent_foobar.tags.append(tag)
db.session.commit()
agent_id = agent_foobar.id
db.session.remove()
agent = Agent.query.filter_by(id=agent_id).first()
self.assertIsNotNone(agent)
tags.sort()
agent_tags = list(str(tag.tag) for tag in agent.tags)
agent_tags.sort()
self.assertListEqual(agent_tags, tags)
def update_tags_in_job(job_id):
job = Job.query.filter_by(id=job_id).first()
if not job:
return (render_template("pyfarm/error.html",
error="Job %s not found" % job_id), NOT_FOUND)
tagnames = request.form["tags"].split(" ")
tagnames = [x.strip() for x in tagnames if not x == ""]
tags = []
for name in tagnames:
tag = Tag.query.filter_by(tag=name).first()
if not tag:
tag = Tag(tag=name)
db.session.add(tag)
tags.append(tag)
job.tags = tags
db.session.add(job)
db.session.commit()
flash("Tags for job %s have been updated." % job.title)
return redirect(url_for("single_job_ui", job_id=job.id), SEE_OTHER)
def post(self):
"""
A ``POST`` to this endpoint will do one of two things:
* create a new tag and return the row
* return the row for an existing tag
Tags only have one column, the tag name. Two tags are automatically
considered equal if the tag names are equal.
.. http:post:: /api/v1/tags/ HTTP/1.1
**Request**
.. sourcecode:: http
POST /api/v1/tags/ HTTP/1.1
Accept: application/json
{
"tag": "interesting"
}
**Response (new tag create)**
.. sourcecode:: http
HTTP/1.1 201 CREATED
Content-Type: application/json
{
"id": 1,
"tag": "interesting"
}
**Request**
.. sourcecode:: http
POST /api/v1/tags/ HTTP/1.1
Accept: application/json
{
"tag": "interesting"
}
**Response (existing tag returned)**
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"id": 1,
"tag": "interesting"
}
:statuscode 200: an existing tag was found and returned
:statuscode 201: a new tag was created
:statuscode 400: there was something wrong with the request (such as
invalid columns being included)
"""
existing_tag = Tag.query.filter_by(tag=g.json["tag"]).first()
if existing_tag:
# No update needed, because Tag only has that one column
return jsonify(existing_tag.to_dict()), OK
else:
new_tag = Tag(**g.json)
db.session.add(new_tag)
db.session.commit()
tag_data = new_tag.to_dict(unpack_relationships=("agents", "jobs"))
logger.info("created tag %s: %r", new_tag.id, tag_data)
return jsonify(tag_data), CREATED
def put(self, tagname=None):
"""
A ``PUT`` to this endpoint will create a new tag under the given URI.
If a tag already exists under that URI, it will be deleted, then
recreated.
Note that when overwriting a tag like that, all relations that are not
explicitly specified here will be deleted
You can optionally specify a list of agents or jobs relations as
integers in the request data.
You should only call this by id for overwriting an existing tag or if you
have a reserved tag id. There is currently no way to reserve a tag id.
.. http:put:: /api/v1/tags/<str:tagname> HTTP/1.1
**Request**
.. sourcecode:: http
PUT /api/v1/tags/interesting HTTP/1.1
Accept: application/json
{
"tag": "interesting"
}
**Response**
.. sourcecode:: http
HTTP/1.1 201 CREATED
Content-Type: application/json
{
"id": 1,
"tag": "interesting"
}
**Request**
.. sourcecode:: http
PUT /api/v1/tags/interesting HTTP/1.1
Accept: application/json
{
"tag": "interesting",
"agents": [1]
"jobs": []
}
**Response**
.. sourcecode:: http
HTTP/1.1 201 CREATED
Content-Type: application/json
{
"id": 1,
"tag": "interesting"
}
:statuscode 201: a new tag was created
:statuscode 400: there was something wrong with the request (such as
invalid columns being included)
:statuscode 404: a referenced agent or job does not exist
"""
if isinstance(tagname, int):
tag = Tag.query.filter_by(id=tagname).first()
if "tag" in g.json and g.json["tag"] != tag.tag:
error = "tag name retrieved for %s does not match tag " \
"name in request" % tagname
return jsonify(error=error), BAD_REQUEST
g.json.setdefault("tag", tag.tag)
elif isinstance(tagname, STRING_TYPES):
g.json.setdefault("tag", tagname)
if g.json["tag"] != tagname:
return jsonify(error="`tag` in data must be equal to the "
"tag in the requested url"), BAD_REQUEST
tag = Tag.query.filter_by(tag=g.json["tag"]).first()
# If tag exists, delete it before recreating it
if tag:
logger.debug(
"tag %s will be replaced with %r on commit", tag.tag, g.json)
db.session.delete(tag)
db.session.flush()
agents = []
if "agents" in g.json:
agent_ids = g.json.pop("agents", [])
if not isinstance(agent_ids, list):
return jsonify(error="agents must be a list"), BAD_REQUEST
try:
agent_ids = list(map(UUID, agent_ids))
except (ValueError, AttributeError):
return jsonify(error="All agent ids must be UUIDs"), BAD_REQUEST
# find all models matching the request id(s)
agents = Agent.query.filter(Agent.id.in_(agent_ids)).all()
# make sure all those ids were actually found
missing_agents = set(agent_ids) - set(agent.id for agent in agents)
if missing_agents:
return jsonify(
error="agent(s) not found: %s" % missing_agents), NOT_FOUND
jobs = []
if "jobs" in g.json:
job_ids = g.json.pop("jobs", [])
if not isinstance(job_ids, list):
return jsonify(error="jobs must be a list"), BAD_REQUEST
# make sure all ids provided are ints
if not all(isinstance(job_id, int) for job_id in job_ids):
return jsonify(
error="all job ids must be integers"), BAD_REQUEST
# find all models matching the request id(s)
jobs = Job.query.filter(Agent.id.in_(job_ids)).all()
# make sure all those ids were actually found
missing_jobs = set(job_ids) - set(job.id for job in jobs)
if missing_jobs:
return jsonify(
error="job(s) not found: %s" % missing_jobs), NOT_FOUND
new_tag = Tag(**g.json)
if isinstance(tagname, int):
new_tag.id = tagname
new_tag.agents = agents
new_tag.jobs = jobs
logger.info("creating tag %s: %r", new_tag.tag, new_tag.to_dict())
db.session.add(new_tag)
db.session.commit()
return (jsonify(new_tag.to_dict(unpack_relationships=("agents", "jobs"))),
CREATED)
def post(self, agent_id):
"""
Update an agent's columns with new information by merging the provided
data with the agent's current definition in the database.
.. http:post:: /api/v1/agents/(str:agent_id) HTTP/1.1
**Request**
.. sourcecode:: http
POST /api/v1/agents/29d466a5-34f8-408a-b613-e6c2715077a0 HTTP/1.1
Accept: application/json
{"ram": 1234}
**Response**
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"cpu_allocation": 1.0,
"cpus": 14,
"use_address": 311,
"free_ram": 133,
"time_offset": 0,
"hostname": "agent1",
"id": "29d466a5-34f8-408a-b613-e6c2715077a0",
"ip": "10.196.200.115",
"port": 64994,
"ram": 1234,
"ram_allocation": 0.8,
"state": "running",
"remote_ip": "10.196.200.115"
}
:statuscode 200:
no error
:statuscode 400:
something within the request is invalid
:statuscode 404:
no agent could be found using the given id
"""
agent = Agent.query.filter_by(id=agent_id).first()
if agent is None:
return jsonify(error="Agent %s not found" % agent_id), NOT_FOUND
if ("remote_ip" not in g.json and request.headers.get(
"User-Agent", "") == "PyFarm/1.0 (agent)"):
g.json["remote_ip"] = request.remote_addr
farm_name = g.json.pop("farm_name", None)
if farm_name and farm_name != OUR_FARM_NAME:
return jsonify(error="Wrong farm name"), BAD_REQUEST
current_assignments = g.json.pop("current_assignments", None)
mac_addresses = g.json.pop("mac_addresses", None)
# TODO return BAD_REQUEST on bad mac addresses
if mac_addresses is not None:
mac_addresses = [
x.lower() for x in mac_addresses if MAC_RE.match(x)
]
gpus = g.json.pop("gpus", None)
disks = g.json.pop("disks", None)
tags = g.json.pop("tags", None)
modified = {}
try:
items = g.json.iteritems
except AttributeError:
items = g.json.items
state = g.json.pop("state", None)
if state and agent.state != _AgentState.DISABLED:
agent.state = state
modified["state"] = state
for key, value in items():
if value != getattr(agent, key):
try:
setattr(agent, key, value)
# There may be something wrong with one of the fields
# that's causing our sqlalchemy model to raise a ValueError.
except ValueError as e:
return jsonify(error=str(e)), BAD_REQUEST
modified[key] = value
agent.last_heard_from = datetime.utcnow()
if "upgrade_to" in modified:
update_agent.delay(agent.id)
if mac_addresses is not None:
modified["mac_addresses"] = mac_addresses
for existing_address in agent.mac_addresses:
if existing_address.mac_address.lower() not in mac_addresses:
logger.debug(
"Existing address %s is not in supplied "
"mac addresses, for agent %s, removing it.",
existing_address.mac_address, agent.hostname)
agent.mac_addresses.remove(existing_address)
else:
mac_addresses.remove(existing_address.mac_address.lower())
for new_address in mac_addresses:
mac_address = AgentMacAddress(agent=agent,
mac_address=new_address)
db.session.add(mac_address)
if gpus is not None:
modified["gpus"] = gpus
for existing_gpu in agent.gpus:
if existing_gpu.fullname not in gpus:
logger.debug(
"Existing gpu %s is not in supplied "
"gpus, for agent %s, removing it.",
existing_gpu.fullname, agent.hostname)
agent.gpus.remove(existing_gpu)
else:
gpus.remove(existing_gpu.fullname)
for gpu_name in gpus:
gpu = GPU.query.filter_by(fullname=gpu_name).first()
if not gpu:
gpu = GPU(fullname=gpu_name)
db.session.add(gpu)
agent.gpus.append(gpu)
if disks is not None:
for old_disk in agent.disks:
db.session.delete(old_disk)
for disk_dict in disks:
disk = AgentDisk(agent=agent,
mountpoint=disk_dict["mountpoint"],
size=disk_dict["size"],
free=disk_dict["free"])
db.session.add(disk)
if tags is not None:
modified["tags"] = tags
for existing_tag in agent.tags:
if existing_tag.tag not in tags:
logger.debug(
"Existing tag %s is not in supplied "
"tags, for agent %s, removing it.", existing_tag.tag,
agent.hostname)
agent.tags.remove(existing_tag)
else:
tags.remove(existing_tag.tag)
for tag_name in tags:
tag = Tag.query.filter_by(tag=tag_name).first()
if not tag:
tag = Tag(tag=tag_name)
db.session.add(tag)
agent.tags.append(tag)
# TODO Only do that if this is really the agent speaking to us.
failed_tasks = []
if (current_assignments is not None
and agent.state != AgentState.OFFLINE):
failed_tasks = fail_missing_assignments(agent, current_assignments)
logger.debug("Updated agent %r: %r", agent.id, modified)
db.session.add(agent)
db.session.commit()
for task in failed_tasks:
task.job.update_state()
if agent.state == _AgentState.OFFLINE:
for task in agent.tasks.filter(Task.state != WorkState.DONE,
Task.state != WorkState.FAILED):
task.agent = None
task.state = None
task.job.update_state()
db.session.add(task)
db.session.commit()
assign_tasks_to_agent.delay(agent_id)
return jsonify(agent.to_dict(unpack_relationships=["tags"])), OK