def test_unique(self): job = Job() tagA = Tag() tagB = Tag() tagA.jobs = [job] tagA.tag = "foo0" tagB.jobs = [job] tagB.tag = "foo1" db.session.add_all([job, tagA, tagB]) with self.assertRaises(DatabaseError): db.session.commit()
def test_null(self): with self.assertRaises(DatabaseError): model = Tag() db.session.add(model) db.session.commit() db.session.remove() with self.assertRaises(DatabaseError): tag = Tag() tag.tag = "foo789" db.session.add(model) db.session.commit()
def test_insert(self): # A job can not be created without a jobtype, create one first jobtype = JobType() jobtype.name = "foo" jobtype.description = "this is a job type" jobtype.classname = "Foobar" jobtype.code = dedent(""" class Foobar(JobType): pass""").encode("utf-8") jobtype.mode = JobTypeLoadMode.OPEN db.session.add(jobtype) job = Job() job.job_type = jobtype tag = Tag() tag.jobs = [job] tag.tag = "foo456" db.session.add_all([tag, job]) db.session.commit() model_id = tag.id job_id = job.id db.session.remove() result = Tag.query.filter_by(id=model_id).first() self.assertEqual(result.tag, "foo456") self.assertEqual(result.jobs[0].id, job_id)
def add_tag_on_jobs(): job_ids = request.form.getlist("job_id") tag_name = request.form["tag"].strip() tag = Tag.query.filter_by(tag=tag_name).first() if not tag: tag = Tag(tag=tag_name) db.session.add(tag) for job_id in job_ids: job = Job.query.filter_by(id=job_id).first() if not job: return (render_template("pyfarm/error.html", error="Job %s not found" % job_id), NOT_FOUND) if tag not in job.tags: job.tags.append(tag) db.session.add(job) db.session.commit() flash("Tag %s has been added to selected jobs." % tag_name) if "next" in request.args: return redirect(request.args.get("next"), SEE_OTHER) else: return redirect(url_for("jobs_index_ui"), SEE_OTHER)
def test_insert(self): # A job can not be created without a jobtype, create one first jobtype = JobType() jobtype.name = "foo" jobtype.description = "this is a job type" jobtype_version = JobTypeVersion() jobtype_version.jobtype = jobtype jobtype_version.version = 1 jobtype_version.classname = "Foobar" jobtype_version.code = (""" class Foobar(JobType): pass""").encode("utf-8") db.session.add(jobtype_version) queue = JobQueue() queue.name = "FooQueue" job = Job() job.title = "Test Job" job.jobtype_version = jobtype_version job.queue = queue tag = Tag() tag.jobs = [job] tag.tag = "foo456" db.session.add_all([tag, job]) db.session.commit() model_id = tag.id job_id = job.id db.session.remove() result = Tag.query.filter_by(id=model_id).first() self.assertEqual(result.tag, "foo456") self.assertEqual(result.jobs[0].id, job_id)
def update_tag_requirements_in_job(job_id): job = Job.query.filter_by(id=job_id).first() if not job: return (render_template("pyfarm/error.html", error="Job %s not found" % job_id), NOT_FOUND) tagnames = request.form["tag_requirements"].split(" ") tagnames = [x.strip() for x in tagnames if not x == ""] job.tag_requirements = [] for name in tagnames: negate = False if name.startswith("-"): if len(name) < 2: return (render_template("pyfarm/error.html", error="Tag too short: %s" % name), NOT_FOUND) negate = True name = name[1:] tag = Tag.query.filter_by(tag=name).first() if not tag: tag = Tag(tag=name) db.session.add(tag) tag_requirement = JobTagRequirement(job=job, tag=tag, negate=negate) db.session.add(tag_requirement) db.session.commit() flash("Tag Requirements for job %s have been updated." % job.title) return redirect(url_for("single_job_ui", job_id=job.id), SEE_OTHER)
def update_tags_in_agent(agent_id): agent = Agent.query.filter_by(id=agent_id).first() if not agent: return (render_template( "pyfarm/error.html", error="Agent %s not found" % agent_id), NOT_FOUND) tagnames = request.form["tags"].split(" ") tagnames = [x.strip() for x in tagnames if not x == ""] tags = [] for name in tagnames: tag = Tag.query.filter_by(tag=name).first() if not tag: tag = Tag(tag=name) db.session.add(tag) tags.append(tag) agent.tags = tags db.session.add(agent) db.session.commit() flash("Tags for agent %s have been updated." % agent.hostname) return redirect(url_for("single_agent_ui", agent_id=agent.id), SEE_OTHER)
def test_tags_validation(self): for agent_foobar in self.models(limit=1): tag = Tag() tag.agent = [agent_foobar] tag.tag = "foo123" db.session.add(tag) db.session.commit() self.assertEqual(tag.tag, "foo123")
def test_tags_validation_error(self): for agent_foobar in self.models(limit=1): tag = Tag() tag.agents = [agent_foobar] tag.tag = None with self.assertRaises(DatabaseError): db.session.add(tag) db.session.commit()
def add_tag_requirement_on_jobs(): job_ids = request.form.getlist("job_id") tag_name = request.form["tag"].strip() negate = False if tag_name[0] == "-": if len(tag_name) < 2: return (render_template( "pyfarm/error.html", error="Tag must be at least one character long"), NOT_FOUND) else: negate = True tag_name = tag_name[1:] tag = Tag.query.filter_by(tag=tag_name).first() if not tag: tag = Tag(tag=tag_name) db.session.add(tag) for job_id in job_ids: job = Job.query.filter_by(id=job_id).first() if not job: return (render_template("pyfarm/error.html", error="Job %s not found" % job_id), NOT_FOUND) tag_requirement = JobTagRequirement.query.filter_by(job=job, tag=tag).first() if not tag_requirement: tag_requirement = JobTagRequirement(job=job, tag=tag, negate=negate) db.session.add(tag_requirement) elif tag_requirement.negate != negate: tag_requirement.negate = negate db.session.add(tag_requirement) db.session.commit() flash("Tag requirement %s has been added to selected jobs." % tag_name) if "next" in request.args: return redirect(request.args.get("next"), SEE_OTHER) else: return redirect(url_for("jobs_index_ui"), SEE_OTHER)
def test_tags(self): for agent_foobar in self.models(limit=1): db.session.add(agent_foobar) tags = [] rand = lambda: uuid.uuid4().hex for tag_name in (rand(), rand(), rand()): tag = Tag() tag.tag = tag_name tags.append(tag_name) agent_foobar.tags.append(tag) db.session.commit() agent_id = agent_foobar.id db.session.remove() agent = Agent.query.filter_by(id=agent_id).first() self.assertIsNotNone(agent) tags.sort() agent_tags = list(str(tag.tag) for tag in agent.tags) agent_tags.sort() self.assertListEqual(agent_tags, tags)
def update_tags_in_job(job_id): job = Job.query.filter_by(id=job_id).first() if not job: return (render_template("pyfarm/error.html", error="Job %s not found" % job_id), NOT_FOUND) tagnames = request.form["tags"].split(" ") tagnames = [x.strip() for x in tagnames if not x == ""] tags = [] for name in tagnames: tag = Tag.query.filter_by(tag=name).first() if not tag: tag = Tag(tag=name) db.session.add(tag) tags.append(tag) job.tags = tags db.session.add(job) db.session.commit() flash("Tags for job %s have been updated." % job.title) return redirect(url_for("single_job_ui", job_id=job.id), SEE_OTHER)
def post(self): """ A ``POST`` to this endpoint will do one of two things: * create a new tag and return the row * return the row for an existing tag Tags only have one column, the tag name. Two tags are automatically considered equal if the tag names are equal. .. http:post:: /api/v1/tags/ HTTP/1.1 **Request** .. sourcecode:: http POST /api/v1/tags/ HTTP/1.1 Accept: application/json { "tag": "interesting" } **Response (new tag create)** .. sourcecode:: http HTTP/1.1 201 CREATED Content-Type: application/json { "id": 1, "tag": "interesting" } **Request** .. sourcecode:: http POST /api/v1/tags/ HTTP/1.1 Accept: application/json { "tag": "interesting" } **Response (existing tag returned)** .. sourcecode:: http HTTP/1.1 200 OK Content-Type: application/json { "id": 1, "tag": "interesting" } :statuscode 200: an existing tag was found and returned :statuscode 201: a new tag was created :statuscode 400: there was something wrong with the request (such as invalid columns being included) """ existing_tag = Tag.query.filter_by(tag=g.json["tag"]).first() if existing_tag: # No update needed, because Tag only has that one column return jsonify(existing_tag.to_dict()), OK else: new_tag = Tag(**g.json) db.session.add(new_tag) db.session.commit() tag_data = new_tag.to_dict(unpack_relationships=("agents", "jobs")) logger.info("created tag %s: %r", new_tag.id, tag_data) return jsonify(tag_data), CREATED
def put(self, tagname=None): """ A ``PUT`` to this endpoint will create a new tag under the given URI. If a tag already exists under that URI, it will be deleted, then recreated. Note that when overwriting a tag like that, all relations that are not explicitly specified here will be deleted You can optionally specify a list of agents or jobs relations as integers in the request data. You should only call this by id for overwriting an existing tag or if you have a reserved tag id. There is currently no way to reserve a tag id. .. http:put:: /api/v1/tags/<str:tagname> HTTP/1.1 **Request** .. sourcecode:: http PUT /api/v1/tags/interesting HTTP/1.1 Accept: application/json { "tag": "interesting" } **Response** .. sourcecode:: http HTTP/1.1 201 CREATED Content-Type: application/json { "id": 1, "tag": "interesting" } **Request** .. sourcecode:: http PUT /api/v1/tags/interesting HTTP/1.1 Accept: application/json { "tag": "interesting", "agents": [1] "jobs": [] } **Response** .. sourcecode:: http HTTP/1.1 201 CREATED Content-Type: application/json { "id": 1, "tag": "interesting" } :statuscode 201: a new tag was created :statuscode 400: there was something wrong with the request (such as invalid columns being included) :statuscode 404: a referenced agent or job does not exist """ if isinstance(tagname, int): tag = Tag.query.filter_by(id=tagname).first() if "tag" in g.json and g.json["tag"] != tag.tag: error = "tag name retrieved for %s does not match tag " \ "name in request" % tagname return jsonify(error=error), BAD_REQUEST g.json.setdefault("tag", tag.tag) elif isinstance(tagname, STRING_TYPES): g.json.setdefault("tag", tagname) if g.json["tag"] != tagname: return jsonify(error="`tag` in data must be equal to the " "tag in the requested url"), BAD_REQUEST tag = Tag.query.filter_by(tag=g.json["tag"]).first() # If tag exists, delete it before recreating it if tag: logger.debug( "tag %s will be replaced with %r on commit", tag.tag, g.json) db.session.delete(tag) db.session.flush() agents = [] if "agents" in g.json: agent_ids = g.json.pop("agents", []) if not isinstance(agent_ids, list): return jsonify(error="agents must be a list"), BAD_REQUEST try: agent_ids = list(map(UUID, agent_ids)) except (ValueError, AttributeError): return jsonify(error="All agent ids must be UUIDs"), BAD_REQUEST # find all models matching the request id(s) agents = Agent.query.filter(Agent.id.in_(agent_ids)).all() # make sure all those ids were actually found missing_agents = set(agent_ids) - set(agent.id for agent in agents) if missing_agents: return jsonify( error="agent(s) not found: %s" % missing_agents), NOT_FOUND jobs = [] if "jobs" in g.json: job_ids = g.json.pop("jobs", []) if not isinstance(job_ids, list): return jsonify(error="jobs must be a list"), BAD_REQUEST # make sure all ids provided are ints if not all(isinstance(job_id, int) for job_id in job_ids): return jsonify( error="all job ids must be integers"), BAD_REQUEST # find all models matching the request id(s) jobs = Job.query.filter(Agent.id.in_(job_ids)).all() # make sure all those ids were actually found missing_jobs = set(job_ids) - set(job.id for job in jobs) if missing_jobs: return jsonify( error="job(s) not found: %s" % missing_jobs), NOT_FOUND new_tag = Tag(**g.json) if isinstance(tagname, int): new_tag.id = tagname new_tag.agents = agents new_tag.jobs = jobs logger.info("creating tag %s: %r", new_tag.tag, new_tag.to_dict()) db.session.add(new_tag) db.session.commit() return (jsonify(new_tag.to_dict(unpack_relationships=("agents", "jobs"))), CREATED)
def post(self, agent_id): """ Update an agent's columns with new information by merging the provided data with the agent's current definition in the database. .. http:post:: /api/v1/agents/(str:agent_id) HTTP/1.1 **Request** .. sourcecode:: http POST /api/v1/agents/29d466a5-34f8-408a-b613-e6c2715077a0 HTTP/1.1 Accept: application/json {"ram": 1234} **Response** .. sourcecode:: http HTTP/1.1 200 OK Content-Type: application/json { "cpu_allocation": 1.0, "cpus": 14, "use_address": 311, "free_ram": 133, "time_offset": 0, "hostname": "agent1", "id": "29d466a5-34f8-408a-b613-e6c2715077a0", "ip": "10.196.200.115", "port": 64994, "ram": 1234, "ram_allocation": 0.8, "state": "running", "remote_ip": "10.196.200.115" } :statuscode 200: no error :statuscode 400: something within the request is invalid :statuscode 404: no agent could be found using the given id """ agent = Agent.query.filter_by(id=agent_id).first() if agent is None: return jsonify(error="Agent %s not found" % agent_id), NOT_FOUND if ("remote_ip" not in g.json and request.headers.get( "User-Agent", "") == "PyFarm/1.0 (agent)"): g.json["remote_ip"] = request.remote_addr farm_name = g.json.pop("farm_name", None) if farm_name and farm_name != OUR_FARM_NAME: return jsonify(error="Wrong farm name"), BAD_REQUEST current_assignments = g.json.pop("current_assignments", None) mac_addresses = g.json.pop("mac_addresses", None) # TODO return BAD_REQUEST on bad mac addresses if mac_addresses is not None: mac_addresses = [ x.lower() for x in mac_addresses if MAC_RE.match(x) ] gpus = g.json.pop("gpus", None) disks = g.json.pop("disks", None) tags = g.json.pop("tags", None) modified = {} try: items = g.json.iteritems except AttributeError: items = g.json.items state = g.json.pop("state", None) if state and agent.state != _AgentState.DISABLED: agent.state = state modified["state"] = state for key, value in items(): if value != getattr(agent, key): try: setattr(agent, key, value) # There may be something wrong with one of the fields # that's causing our sqlalchemy model to raise a ValueError. except ValueError as e: return jsonify(error=str(e)), BAD_REQUEST modified[key] = value agent.last_heard_from = datetime.utcnow() if "upgrade_to" in modified: update_agent.delay(agent.id) if mac_addresses is not None: modified["mac_addresses"] = mac_addresses for existing_address in agent.mac_addresses: if existing_address.mac_address.lower() not in mac_addresses: logger.debug( "Existing address %s is not in supplied " "mac addresses, for agent %s, removing it.", existing_address.mac_address, agent.hostname) agent.mac_addresses.remove(existing_address) else: mac_addresses.remove(existing_address.mac_address.lower()) for new_address in mac_addresses: mac_address = AgentMacAddress(agent=agent, mac_address=new_address) db.session.add(mac_address) if gpus is not None: modified["gpus"] = gpus for existing_gpu in agent.gpus: if existing_gpu.fullname not in gpus: logger.debug( "Existing gpu %s is not in supplied " "gpus, for agent %s, removing it.", existing_gpu.fullname, agent.hostname) agent.gpus.remove(existing_gpu) else: gpus.remove(existing_gpu.fullname) for gpu_name in gpus: gpu = GPU.query.filter_by(fullname=gpu_name).first() if not gpu: gpu = GPU(fullname=gpu_name) db.session.add(gpu) agent.gpus.append(gpu) if disks is not None: for old_disk in agent.disks: db.session.delete(old_disk) for disk_dict in disks: disk = AgentDisk(agent=agent, mountpoint=disk_dict["mountpoint"], size=disk_dict["size"], free=disk_dict["free"]) db.session.add(disk) if tags is not None: modified["tags"] = tags for existing_tag in agent.tags: if existing_tag.tag not in tags: logger.debug( "Existing tag %s is not in supplied " "tags, for agent %s, removing it.", existing_tag.tag, agent.hostname) agent.tags.remove(existing_tag) else: tags.remove(existing_tag.tag) for tag_name in tags: tag = Tag.query.filter_by(tag=tag_name).first() if not tag: tag = Tag(tag=tag_name) db.session.add(tag) agent.tags.append(tag) # TODO Only do that if this is really the agent speaking to us. failed_tasks = [] if (current_assignments is not None and agent.state != AgentState.OFFLINE): failed_tasks = fail_missing_assignments(agent, current_assignments) logger.debug("Updated agent %r: %r", agent.id, modified) db.session.add(agent) db.session.commit() for task in failed_tasks: task.job.update_state() if agent.state == _AgentState.OFFLINE: for task in agent.tasks.filter(Task.state != WorkState.DONE, Task.state != WorkState.FAILED): task.agent = None task.state = None task.job.update_state() db.session.add(task) db.session.commit() assign_tasks_to_agent.delay(agent_id) return jsonify(agent.to_dict(unpack_relationships=["tags"])), OK