Exemplo n.º 1
0
    def post(self):
        """
        A ``POST`` to this endpoint will either create or update an existing
        agent.  The ``port`` and ``id`` columns will determine if an
        agent already exists.

            * If an agent is found matching the ``port`` and ``id``
              columns from the request the existing model will be updated and
              the resulting data and the ``OK`` code will be returned.

            * If we don't find an agent matching the ``port`` and ``id``
              however a new agent will be created and the resulting data and the
              ``CREATED`` code will be returned.

        .. note::
            The ``remote_ip`` field is not required and should typically
            not be included in a request.  When not provided ``remote_ip``
            is be populated by the server based off of the ip of the
            incoming request.  Providing ``remote_ip`` in your request
            however will override this behavior.

        .. http:post:: /api/v1/agents/ HTTP/1.1

            **Request**

            .. sourcecode:: http

                POST /api/v1/agents/ HTTP/1.1
                Accept: application/json

                {
                    "cpu_allocation": 1.0,
                    "cpus": 14,
                    "free_ram": 133,
                    "hostname": "agent1",
                    "id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
                    "remote_ip": "10.196.200.115",
                    "port": 64994,
                    "ram": 2157,
                    "ram_allocation": 0.8,
                    "state": 8
                 }

            **Response (agent created)**

            .. sourcecode:: http

                HTTP/1.1 201 CREATED
                Content-Type: application/json

                {
                    "cpu_allocation": 1.0,
                    "cpus": 14,
                    "use_address": "remote",
                    "free_ram": 133,
                    "time_offset": 0,
                    "hostname": "agent1",
                    "id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
                    "port": 64994,
                    "ram": 2157,
                    "ram_allocation": 0.8,
                    "state": "online",
                    "remote_ip": "10.196.200.115"
                 }

            **Response (existing agent updated)**

            .. sourcecode:: http

                HTTP/1.1 200 OK
                Content-Type: application/json

                {
                    "cpu_allocation": 1.0,
                    "cpus": 14,
                    "use_address": "remote",
                    "free_ram": 133,
                    "time_offset": 0,
                    "hostname": "agent1",
                    "id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
                    "port": 64994,
                    "ram": 2157,
                    "ram_allocation": 0.8,
                    "state": "online",
                    "remote_ip": "10.196.200.115"
                 }
 
        :statuscode 201:
            a new agent was created

        :statuscode 200:
            an existing agent is updated with data from the request

        :statuscode 400:
            there was something wrong with the request (such as
            invalid columns being included)
        """
        # Read in and convert the id field
        try:
            g.json["id"] = uuid.UUID(g.json["id"])
        except KeyError:
            return jsonify(error="`id` not provided"), BAD_REQUEST

        # If the remote user agent is not "PyFarm/1.0 (agent)", this is not an
        # announce by the agent itself. It could be some other client editing
        # the agent.
        if request.headers.get("User-Agent", "") == "PyFarm/1.0 (agent)":
            # Set remote_ip if it did not come in with the request
            g.json.setdefault("remote_ip", request.remote_addr)

        farm_name = g.json.pop("farm_name", None)
        if farm_name and farm_name != OUR_FARM_NAME:
            return jsonify(error="Wrong farm name"), BAD_REQUEST

        current_assignments = g.json.pop("current_assignments", None)
        mac_addresses = g.json.pop("mac_addresses", None)
        # TODO return BAD_REQUEST on bad mac addresses
        if mac_addresses is not None:
            mac_addresses = [x.lower() for x in mac_addresses if MAC_RE.match(x)]

        gpus = g.json.pop("gpus", None)
        disks = g.json.pop("disks", None)
        state = g.json.pop("state", None)

        agent = Agent.query.filter_by(
            port=g.json["port"], id=g.json["id"]).first()

        if agent is None:
            try:
                agent = Agent(**g.json)

            # There may be something wrong with one of the fields
            # that's causing our sqlalchemy model raise a ValueError.
            except ValueError as e:
                return jsonify(error=str(e)), BAD_REQUEST

            default_versions = SoftwareVersion.query.filter_by(default=True)
            for version in default_versions:
                 agent.software_versions.append(version)

            if mac_addresses is not None:
                for address in mac_addresses:
                    mac_address = AgentMacAddress(agent=agent,
                                                  mac_address=address)
                    db.session.add(mac_address)

            if gpus is not None:
                for gpu_name in gpus:
                    gpu = GPU.query.filter_by(
                        fullname=gpu_name).first()
                    if not gpu:
                        gpu = GPU(fullname=gpu_name)
                        db.session.add(gpu)
                    agent.gpus.append(gpu)

            if disks is not None:
                for disk_dict in disks:
                    disk = AgentDisk(agent=agent,
                                     mountpoint=disk_dict["mountpoint"],
                                     size=disk_dict["size"],
                                     free=disk_dict["free"])
                    db.session.add(disk)

            if state is not None:
                agent.state = state

            db.session.add(agent)

            try:
                db.session.commit()

            except Exception as e:
                e = e.args[0].lower()
                error = "Unhandled error: %s.  This is often an issue " \
                        "with the agent's data for `ip`, `hostname` and/or " \
                        "`port` not being unique enough.  In other cases " \
                        "this can sometimes happen if the underlying " \
                        "database driver is either non-compliant with " \
                        "expectations or we've encountered a database error " \
                        "that we don't know how to handle yet.  If the " \
                        "latter is the case, please report this as a bug." % e

                return jsonify(error=error), INTERNAL_SERVER_ERROR

            else:
                agent_data = agent.to_dict(unpack_relationships=["tags"])
                logger.info("Created agent %r: %r", agent.id, agent_data)
                assign_tasks.delay()
                return jsonify(agent_data), CREATED

        else:
            updated = False

            for key in g.json.copy():
                value = g.json.pop(key)

                if not hasattr(agent, key):
                    return jsonify(
                        error="Agent has no such column `%s`" % key), \
                           BAD_REQUEST

                if getattr(agent, key) != value:
                    try:
                        setattr(agent, key, value)

                    except Exception as e:
                        return jsonify(
                            error="Error while setting `%s`: %s" % (key, e)), \
                               BAD_REQUEST
                    else:
                        updated = True

            if mac_addresses is not None:
                updated = True
                for existing_address in agent.mac_addresses:
                    if existing_address.mac_address.lower() not in mac_addresses:
                        logger.debug("Existing address %s is not in supplied "
                                     "mac addresses, for agent %s, removing it.",
                                     existing_address.mac_address,
                                     agent.hostname)
                        agent.mac_addresses.remove(existing_address)
                    else:
                        mac_addresses.remove(
                            existing_address.mac_address.lower())

                for new_address in mac_addresses:
                    mac_address = AgentMacAddress(
                        agent=agent, mac_address=new_address)
                    db.session.add(mac_address)

            if gpus is not None:
                updated = True
                for existing_gpu in agent.gpus:
                    if existing_gpu.fullname not in gpus:
                        logger.debug("Existing gpu %s is not in supplied "
                                     "gpus, for agent %s, removing it.",
                                     existing_address.mac_address,
                                     agent.hostname)
                        agent.gpus.remove(existing_gpu)
                    else:
                        gpus.remove(existing_gpu.fullname)

                for gpu_name in gpus:
                    gpu = GPU.query.filter_by(fullname=gpu_name).first()
                    if not gpu:
                        gpu = GPU(fullname=gpu_name)
                        db.session.add(gpu)
                    agent.gpus.append(gpu)

            if disks is not None:
                for old_disk in agent.disks:
                    db.session.delete(old_disk)
                for disk_dict in disks:
                    disk = AgentDisk(agent=agent,
                                     mountpoint=disk_dict["mountpoint"],
                                     size=disk_dict["size"],
                                     free=disk_dict["free"])
                    db.session.add(disk)

            if state is not None and agent.state != _AgentState.DISABLED:
                agent.state = state

            # TODO Only do that if this is really the agent speaking to us.
            failed_tasks = []
            if (current_assignments is not None and
                agent.state != AgentState.OFFLINE):
                    fail_missing_assignments(agent, current_assignments)

            if updated or failed_tasks:
                agent.last_heard_from = datetime.utcnow()
                db.session.add(agent)

                try:
                    db.session.commit()

                except Exception as e:
                    return jsonify(error="Unhandled error: %s" % e), \
                           INTERNAL_SERVER_ERROR

                else:
                    agent_data = agent.to_dict(unpack_relationships=["tags"])
                    logger.info("Updated agent %r: %r", agent.id, agent_data)
                    for task in failed_tasks:
                        task.job.update_state()
                    db.session.commit()
                    assign_tasks.delay()
                    return jsonify(agent_data), OK
Exemplo n.º 2
0
    def post(self):
        """
        A ``POST`` to this endpoint will either create or update an existing
        agent.  The ``port`` and ``id`` columns will determine if an
        agent already exists.

            * If an agent is found matching the ``port`` and ``id``
              columns from the request the existing model will be updated and
              the resulting data and the ``OK`` code will be returned.

            * If we don't find an agent matching the ``port`` and ``id``
              however a new agent will be created and the resulting data and the
              ``CREATED`` code will be returned.

        .. note::
            The ``remote_ip`` field is not required and should typically
            not be included in a request.  When not provided ``remote_ip``
            is be populated by the server based off of the ip of the
            incoming request.  Providing ``remote_ip`` in your request
            however will override this behavior.

        .. http:post:: /api/v1/agents/ HTTP/1.1

            **Request**

            .. sourcecode:: http

                POST /api/v1/agents/ HTTP/1.1
                Accept: application/json

                {
                    "cpu_allocation": 1.0,
                    "cpus": 14,
                    "free_ram": 133,
                    "hostname": "agent1",
                    "id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
                    "remote_ip": "10.196.200.115",
                    "port": 64994,
                    "ram": 2157,
                    "ram_allocation": 0.8,
                    "state": 8
                 }

            **Response (agent created)**

            .. sourcecode:: http

                HTTP/1.1 201 CREATED
                Content-Type: application/json

                {
                    "cpu_allocation": 1.0,
                    "cpus": 14,
                    "use_address": "remote",
                    "free_ram": 133,
                    "time_offset": 0,
                    "hostname": "agent1",
                    "id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
                    "port": 64994,
                    "ram": 2157,
                    "ram_allocation": 0.8,
                    "state": "online",
                    "remote_ip": "10.196.200.115"
                 }

            **Response (existing agent updated)**

            .. sourcecode:: http

                HTTP/1.1 200 OK
                Content-Type: application/json

                {
                    "cpu_allocation": 1.0,
                    "cpus": 14,
                    "use_address": "remote",
                    "free_ram": 133,
                    "time_offset": 0,
                    "hostname": "agent1",
                    "id": "6a0c11df-660f-4c1e-9fb4-5fe2b8cd2437",
                    "port": 64994,
                    "ram": 2157,
                    "ram_allocation": 0.8,
                    "state": "online",
                    "remote_ip": "10.196.200.115"
                 }
 
        :statuscode 201:
            a new agent was created

        :statuscode 200:
            an existing agent is updated with data from the request

        :statuscode 400:
            there was something wrong with the request (such as
            invalid columns being included)
        """
        # Read in and convert the id field
        try:
            g.json["id"] = uuid.UUID(g.json["id"])
        except KeyError:
            return jsonify(error="`id` not provided"), BAD_REQUEST

        # If the remote user agent is not "PyFarm/1.0 (agent)", this is not an
        # announce by the agent itself. It could be some other client editing
        # the agent.
        if request.headers.get("User-Agent", "") == "PyFarm/1.0 (agent)":
            # Set remote_ip if it did not come in with the request
            g.json.setdefault("remote_ip", request.remote_addr)

        farm_name = g.json.pop("farm_name", None)
        if farm_name and farm_name != OUR_FARM_NAME:
            return jsonify(error="Wrong farm name"), BAD_REQUEST

        current_assignments = g.json.pop("current_assignments", None)
        mac_addresses = g.json.pop("mac_addresses", None)
        # TODO return BAD_REQUEST on bad mac addresses
        if mac_addresses is not None:
            mac_addresses = [
                x.lower() for x in mac_addresses if MAC_RE.match(x)
            ]

        gpus = g.json.pop("gpus", None)
        disks = g.json.pop("disks", None)
        state = g.json.pop("state", None)

        agent = Agent.query.filter_by(port=g.json["port"],
                                      id=g.json["id"]).first()

        if agent is None:
            try:
                agent = Agent(**g.json)

            # There may be something wrong with one of the fields
            # that's causing our sqlalchemy model raise a ValueError.
            except ValueError as e:
                return jsonify(error=str(e)), BAD_REQUEST

            default_versions = SoftwareVersion.query.filter_by(default=True)
            for version in default_versions:
                agent.software_versions.append(version)

            if mac_addresses is not None:
                for address in mac_addresses:
                    mac_address = AgentMacAddress(agent=agent,
                                                  mac_address=address)
                    db.session.add(mac_address)

            if gpus is not None:
                for gpu_name in gpus:
                    gpu = GPU.query.filter_by(fullname=gpu_name).first()
                    if not gpu:
                        gpu = GPU(fullname=gpu_name)
                        db.session.add(gpu)
                    agent.gpus.append(gpu)

            if disks is not None:
                for disk_dict in disks:
                    disk = AgentDisk(agent=agent,
                                     mountpoint=disk_dict["mountpoint"],
                                     size=disk_dict["size"],
                                     free=disk_dict["free"])
                    db.session.add(disk)

            if state is not None:
                agent.state = state

            db.session.add(agent)

            try:
                db.session.commit()

            except Exception as e:
                e = e.args[0].lower()
                error = "Unhandled error: %s.  This is often an issue " \
                        "with the agent's data for `ip`, `hostname` and/or " \
                        "`port` not being unique enough.  In other cases " \
                        "this can sometimes happen if the underlying " \
                        "database driver is either non-compliant with " \
                        "expectations or we've encountered a database error " \
                        "that we don't know how to handle yet.  If the " \
                        "latter is the case, please report this as a bug." % e

                return jsonify(error=error), INTERNAL_SERVER_ERROR

            else:
                agent_data = agent.to_dict(unpack_relationships=["tags"])
                logger.info("Created agent %r: %r", agent.id, agent_data)
                assign_tasks.delay()
                return jsonify(agent_data), CREATED

        else:
            updated = False

            for key in g.json.copy():
                value = g.json.pop(key)

                if not hasattr(agent, key):
                    return jsonify(
                        error="Agent has no such column `%s`" % key), \
                           BAD_REQUEST

                if getattr(agent, key) != value:
                    try:
                        setattr(agent, key, value)

                    except Exception as e:
                        return jsonify(
                            error="Error while setting `%s`: %s" % (key, e)), \
                               BAD_REQUEST
                    else:
                        updated = True

            if mac_addresses is not None:
                updated = True
                for existing_address in agent.mac_addresses:
                    if existing_address.mac_address.lower(
                    ) not in mac_addresses:
                        logger.debug(
                            "Existing address %s is not in supplied "
                            "mac addresses, for agent %s, removing it.",
                            existing_address.mac_address, agent.hostname)
                        agent.mac_addresses.remove(existing_address)
                    else:
                        mac_addresses.remove(
                            existing_address.mac_address.lower())

                for new_address in mac_addresses:
                    mac_address = AgentMacAddress(agent=agent,
                                                  mac_address=new_address)
                    db.session.add(mac_address)

            if gpus is not None:
                updated = True
                for existing_gpu in agent.gpus:
                    if existing_gpu.fullname not in gpus:
                        logger.debug(
                            "Existing gpu %s is not in supplied "
                            "gpus, for agent %s, removing it.",
                            existing_address.mac_address, agent.hostname)
                        agent.gpus.remove(existing_gpu)
                    else:
                        gpus.remove(existing_gpu.fullname)

                for gpu_name in gpus:
                    gpu = GPU.query.filter_by(fullname=gpu_name).first()
                    if not gpu:
                        gpu = GPU(fullname=gpu_name)
                        db.session.add(gpu)
                    agent.gpus.append(gpu)

            if disks is not None:
                for old_disk in agent.disks:
                    db.session.delete(old_disk)
                for disk_dict in disks:
                    disk = AgentDisk(agent=agent,
                                     mountpoint=disk_dict["mountpoint"],
                                     size=disk_dict["size"],
                                     free=disk_dict["free"])
                    db.session.add(disk)

            if state is not None and agent.state != _AgentState.DISABLED:
                agent.state = state

            # TODO Only do that if this is really the agent speaking to us.
            failed_tasks = []
            if (current_assignments is not None
                    and agent.state != AgentState.OFFLINE):
                fail_missing_assignments(agent, current_assignments)

            if updated or failed_tasks:
                agent.last_heard_from = datetime.utcnow()
                db.session.add(agent)

                try:
                    db.session.commit()

                except Exception as e:
                    return jsonify(error="Unhandled error: %s" % e), \
                           INTERNAL_SERVER_ERROR

                else:
                    agent_data = agent.to_dict(unpack_relationships=["tags"])
                    logger.info("Updated agent %r: %r", agent.id, agent_data)
                    for task in failed_tasks:
                        task.job.update_state()
                    db.session.commit()
                    assign_tasks.delay()
                    return jsonify(agent_data), OK