Пример #1
0
def setup_app(app):




   # All your initialization code
    bee_db = BeekeeperDB()


    for table_name in ['nodes_log', 'nodes_history', 'beehives']:

        stmt = f'SHOW COLUMNS FROM `{table_name}`'
        logger.debug(f'statement: {stmt}')
        bee_db.cur.execute(stmt)
        rows = bee_db.cur.fetchall()


        table_fields[table_name] = []
        table_fields_index[table_name] ={}
        for row in rows:
            #print(row, flush=True)
            table_fields[table_name].append(row[0])

        for f in range(len(table_fields[table_name])):
            table_fields_index[table_name][table_fields[table_name][f]] = f

    bee_db.close()
    logger.debug(table_fields)
    logger.debug(table_fields_index)


    initialize_test_nodes()
Пример #2
0
def initialize_test_nodes():  # pragma: no cover   this code is not used in production


    test_nodes_file = os.path.join(BASE_KEY_DIR, "test-nodes.txt")
    if not os.path.isfile(test_nodes_file):
        logger.debug(f"File {test_nodes_file} not found. (only needed in testing)")
        return


    logger.debug(f"File {test_nodes_file} found. Load test nodes into DB")

    ### collect info about registered nodes to prevent multiple registrations
    try:
        bee_db = BeekeeperDB()
        node_list = bee_db.list_latest_state()
        bee_db.close()
    except Exception as e:
        raise Exception(f'list_latest_state returned: {str(e)}')


    registered_nodes={}
    node_list_len = len(node_list)
    logger.debug(f"Got {node_list_len} nodes from beekeeper API.")
    for node_object in node_list:
        if "id" not in node_object:
            logger.error("Field id missing")
            continue

        node_id = node_object["id"]
        registered_nodes[node_id]=True


    # check if nodes are registered already
    with open(test_nodes_file) as f:
        for line in f:
            if len(line) <= 1: # skip empty lines
                continue
            if line[0]=="#": # skip comments
                continue

            node_id = line
            logger.debug(f"got node: {node_id}")
            if node_id in registered_nodes:
                logger.debug(f"Node {node_id} already registered.")
                continue

            # register node

            try:
                register_node(node_id, lock_tables=False)  # Locking mysql tables at initialization does not work for some unclear reason
            except Exception as e:
                raise Exception(f"Node registration failed: {str(e)}")

            logger.debug(f"Node {node_id} registered.")


    return
Пример #3
0
    def get(self):
        try:
            bee_db = BeekeeperDB()
            node_state = bee_db.list_latest_state()
            bee_db.close()
        except Exception as e:
            raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        return { "data" : node_state }
Пример #4
0
    def get(self, beehive_id):


        bee_db = BeekeeperDB()
        obj = bee_db.get_beehive(beehive_id)
        if not obj:
            raise Exception(f"Beehive {beehive_id} not found" )


        bee_db.close()
        return jsonify(obj)
Пример #5
0
    def delete(self, node_id):


        try:

            bee_db = BeekeeperDB()
            result_count = bee_db.delete_object( "node_credentials", "id", node_id)
            bee_db.close()
        except Exception as e:
            raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        return jsonify({"deleted": result_count})
Пример #6
0
    def post(self, beehive_id):


        expected_forms = ["tls-key", "tls-cert", "ssh-key", "ssh-pub", "ssh-cert"]

        count_updated = 0
        data={}
        try:

            bee_db = BeekeeperDB()
            obj = bee_db.get_beehive(beehive_id)
            if not obj:
                raise Exception(f"Beehive {beehive_id} not found" )



            for formname in request.files:
                if not formname in expected_forms:
                    raise Exception(f"Formname {formname} not supported" )

            # we could remove this check...
            for formname in expected_forms:
                if not formname in request.files:
                    raise Exception(f"Formname {formname} missing" )


            for formname in request.files:


                formdata = request.files.get(formname).read().decode("utf-8")
                if not formdata:
                    raise Exception(f"Field {formname} empty" )

                data[formname] = formdata
                #logger.debug(f"data: {formname} {data[formname]}")
                #filename = secure_filename(file.filename)
                #logger.debug(f"filename: {filename}")


            for formname in data:

                col_name = formname.replace("-", "_ca_")

                count_updated += bee_db.update_object_field("beehives", col_name, data[formname], "id", beehive_id)

            bee_db.close()
        except Exception as e:
            raise ErrorResponse(f"something failed: {str(e)}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        return jsonify({"modified": count_updated})
Пример #7
0
def post_node_credentials(node_id, private_key, public_key):

    #ssh_key_private", "ssh_key_public
    bee_db = BeekeeperDB()
    post_creds = {"ssh_key_private": private_key, "ssh_key_public": public_key}

    try:
        bee_db.set_node_keypair(node_id, post_creds)
    except Exception as e:
        raise Exception(f"set_node_keypair returned: {str(e)}")

    bee_db.close()

    return
Пример #8
0
    def get(self, node_id):


        try:

            bee_db = BeekeeperDB()
            results = bee_db.get_node_keypair(node_id)
            bee_db.close()
        except Exception as e:
            raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        if not results:
            raise ErrorResponse(f"Not found." , status_code=HTTPStatus.NOT_FOUND )

        return jsonify(results)
Пример #9
0
    def get(self, node_id):
        try:

            bee_db = BeekeeperDB()
            node_state = bee_db.get_node_state(node_id)
            bee_db.close()
        except Exception as e:
            raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        if node_state == None:
            raise ErrorResponse(f"Error: node {node_id} not found")



        return { "data" : node_state }
Пример #10
0
    def get(self):

        view = request.args.get('view', "")

        try:
            bee_db = BeekeeperDB()
            fields = ["id"]
            if view == "full":
                fields=None

            result =  bee_db.get_objects('beehives', fields = fields)
            bee_db.close()
        except Exception as e:
            raise ErrorResponse(f"Error getting list of beehives: { type(e).__name__ }  {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        return jsonify({"data":result})
Пример #11
0
    def post(self):
        try:
#request.get
            postData = request.get_json(force=True, silent=False)

        except Exception as e:

            raise ErrorResponse(f"Error parsing json: { sys.exc_info()[0] }  {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)


        if not "id" in postData:
            raise ErrorResponse(f"Field id is missing" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        if not "key-type" in postData:
            raise ErrorResponse(f"Field key-type is missing" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        beehive_id = postData["id"]
        key_type = postData["key-type"]
        #key_type_args = postData.get("key-type-args", "")

        bee_db = BeekeeperDB()

        #TODO check if beehive already exists

        beehive_obj = {"id": beehive_id}

        #modified = 0
        #updates = {}
        for key in ["key_type", "key_type_args", "rmq_host", "rmq_port" , "upload_host" , "upload_port"]:
            key_dash = key.replace("_", "-")
            if not key_dash in postData:
                continue

            beehive_obj[key] = postData[key_dash]



        modified = bee_db.insert_object("beehives", beehive_obj, force=True)
        bee_db.close()
        return jsonify({"modified": modified})
Пример #12
0
    def post(self, node_id):



        try:
#request.get
            postData = request.get_json(force=True, silent=False)

        except Exception as e:

            raise ErrorResponse(f"Error parsing json: { sys.exc_info()[0] }  {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        if not postData:
            raise ErrorResponse(f"Could not parse json." , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        valid_keys = {"ssh_key_private", "ssh_key_public"}
        expected_keys  = valid_keys

        for key in postData:
            if key not in valid_keys:
                raise ErrorResponse(f"Key {key} not supported" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)

        for key in expected_keys:
            if key not in postData:
                raise ErrorResponse(f"Key {key} missing" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)



        try:

            bee_db = BeekeeperDB()
            results = bee_db.set_node_keypair(node_id, postData)
            bee_db.close()
        except Exception as e:
            raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR)



        return "success"
Пример #13
0
def set_node_beehive(node_id, beehive_id):

    # At this point we already know that beehive exists.
    # Now check if node is already assigned to that beehive
    node_state = None

    try:
        bee_db = BeekeeperDB()
        node_state = bee_db.get_node_state(node_id)
    #except bk_db.ObjectNotFound:
    #    node_state = None
        bee_db.close()
    except Exception as e:
        logger.debug(f"Getting node failed: {str(e)}")
        time.sleep(3)
        pass

    if not node_state:
        raise Exception(f"node {node_id} not found")


    if "beehive" in node_state:
        if node_state["beehive"] == beehive_id:
            logger.debug(f"Node {node_id} is already assigned to beehive {beehive_id}")
            return

    payload = {"node_id": node_id, "source": "beekeeper-register", "operation":"insert", "field_name": "beehive", "field_value": beehive_id}

    #url = f'{BEEKEEPER_DB_API}/log'
    try:
        insert_log(payload, lock_tables=True, force=True, lock_requested_by="set_node_beehive")
        #bk_api_response = requests.post(url,data=json.dumps(payload), timeout=3)
    except Exception as e:
        #raise Exception(f"Error: X Beekeeper DB API ({url}) cannot be reached: {str(e)}")
        raise Exception(f"insert_log returned: {str(e)}")


    return
Пример #14
0
def get_node_keypair(node_id):


    try:
        bee_db = BeekeeperDB()
        return_obj = bee_db.get_node_keypair(node_id)
        bee_db.close()
    except Exception as e:
        raise Exception(f"bee_db.get_node_keypair returned: {str(e)}")

    if not return_obj:
        return None

    if not "ssh_key_private" in return_obj:
        raise Exception(f"Key ssh_key_private missing")

    if not "ssh_key_public" in return_obj:
        raise Exception(f"Key ssh_key_public missing")

    private_key = return_obj["ssh_key_private"]
    public_key = return_obj["ssh_key_public"]

    creds = {"private_key": private_key, "public_key": public_key}
    return creds
Пример #15
0
def deploy_wes(node_id, this_debug, force=False):

    logger.debug("(deploy_wes) determine correct beehive")

    assign_beehive = ""
    bee_db = None
    try:
        bee_db = BeekeeperDB()
        node_state = bee_db.get_node_state(node_id)
    except Exception as e:
        raise Exception(f"finding beehive for node failed: {str(e)}")

    if not node_state:
        raise Exception(f"node {node_id} not found")

    if not "beehive" in node_state:
        raise Exception(f"Node is not assigned to any beehive")

    assign_beehive=node_state["beehive"]
    if assign_beehive == "":
        raise Exception(f"Node is not assigned to any beehive")

    try:
        beehive_obj = bee_db.get_beehive(assign_beehive)
    except Exception as e:
        raise Exception(f"finding beehive for node failed: {str(e)}")

    if not beehive_obj:
        raise Exception(f"Beehive {assign_beehive} unknown" )

    if not isinstance(beehive_obj, dict):
        raise Exception(f"beehive_obj is not a dict")

    beehive_id = beehive_obj.get("id", "")
    if not beehive_id:
        raise Exception(f"beehive_id is missing")

    logger.debug(f"(deploy_wes) using beehive {beehive_id}")


    # first check if kubectl work on the node
    try:
        result_stdout_str ,result_stderr_str, exit_code = node_ssh(node_id, "kubectl get nodes")
    except Exception as e:
        raise Exception(f"node_ssh failed: {str(e)}")

    if exit_code != 0:
        raise Exception(f"ssh failed or kubectl is not yet ready ({result_stderr_str})")


    logger.debug("calling create_ssh_upload_cert")
    try:
        create_ssh_upload_cert(bee_db, node_id, beehive_obj, force=force )
    except Exception as e:
        raise Exception(f"create_ssh_upload_cert failed: {type(e).__name__} {str(e)}")

    logger.debug("calling create_tls_cert_for_node")
    try:
        create_tls_cert_for_node(bee_db, node_id, beehive_obj, force=force)
    except Exception as e:
        raise Exception(f"create_tls_cert_for_node failed: {type(e).__name__} {str(e)}")

    node_keypair = bee_db.get_node_keypair(node_id)

    if not "ssh_key_private" in node_keypair:
        raise Exception(f"ssh_key_private field missing in node_keypair")

    if not "ssh_key_public" in node_keypair:
        raise Exception(f"ssh_key_public field missing in node_keypair")

    ssh_key_private = node_keypair["ssh_key_private"]
    ssh_key_public = node_keypair["ssh_key_public"]

    logger.debug("calling get_node_credentials_all")
    try:
        node_creds = bee_db.get_node_credentials_all(node_id, beehive_id)
    except Exception as e:
        raise Exception(f"get_node_credentials_all failed: {type(e).__name__} {str(e)}")

    bee_db.close()

    if not "tls_cert" in node_creds:
        raise Exception(f"tls_cert field missing in node_creds")
    if not "tls_key" in node_creds:
        raise Exception(f"tls_key field missing in node_creds")

    if not "ssh_upload_cert" in node_creds:
        raise Exception(f"ssh_upload_cert field missing in node_creds")
    ssh_upload_cert = node_creds["ssh_upload_cert"]

    # create and push secret for upload-ssh-key
    upload_secret = kube_secret("wes-beehive-upload-ssh-key", {
        "ssh-key": ssh_key_private,
        "ssh-key.pub": ssh_key_public,
        "ssh-key-cert.pub": ssh_upload_cert,
    })
    kubectl_apply_with_logging(node_id, upload_secret)

    # create and upload tls secret
    tls_secret = kube_secret("wes-beehive-rabbitmq-tls", {
        "cert.pem": node_creds["tls_cert"],
        "key.pem": node_creds["tls_key"],
    })
    kubectl_apply_with_logging(node_id, tls_secret)

    #return {"result": "A"}
    ####################
    # waggle id / host / port config map

    for key in ["rmq_host", "rmq_port", "upload_host", "upload_port"]:
        if not key in beehive_obj:
            raise Exception(f"Beekeeper field {key} missing")

    rmq_host = beehive_obj["rmq_host"]
    rmq_port = beehive_obj["rmq_port"]
    upload_host = beehive_obj["upload_host"]
    upload_port = beehive_obj["upload_port"]

    waggle_ConfigMap = kube_configmap("waggle-config", {
        "WAGGLE_NODE_ID": node_id.lower(),
        "WAGGLE_BEEHIVE_RABBITMQ_HOST": rmq_host,
        "WAGGLE_BEEHIVE_RABBITMQ_PORT": str(rmq_port),
        "WAGGLE_BEEHIVE_UPLOAD_HOST": upload_host,
        "WAGGLE_BEEHIVE_UPLOAD_PORT": str(upload_port),
    })
    kubectl_apply_with_logging(node_id, waggle_ConfigMap)

    ###########################
    # beehive-ssh-ca configmap

    for key in ["ssh-pub", "ssh-cert", "tls-cert"]:
        if not key in beehive_obj:
            available_str = ",".join(list(beehive_obj))
            raise Exception(f"Beekeeper field {key} missing (got: {available_str})")

    ca_ssh_pub = beehive_obj["ssh-pub"]
    ca_ssh_cert = beehive_obj["ssh-cert"]
    ca_tls_cert = beehive_obj["tls-cert"]
    #return {"result": "B"}
    beehive_ssh_ca_ConfigMap = kube_configmap("beehive-ssh-ca", {
        "ca.pub": ca_ssh_pub,
        "ca-cert.pub": ca_ssh_cert,
    })
    kubectl_apply_with_logging(node_id, beehive_ssh_ca_ConfigMap)

    ###########################
    # beehive-tls-ca configmap

    beehive_tls_ca_ConfigMap = kube_configmap("beehive-ca-certificate", {
        "cacert.pem": ca_tls_cert,
    })
    kubectl_apply_with_logging(node_id, beehive_tls_ca_ConfigMap)

    node_private_git_repo_key = "/config/node-private-git-repo-key/node-private-git-repo-key"
    if os.path.exists(node_private_git_repo_key):
        result_stdout , result_stderr, exit_code = scp(node_private_git_repo_key, node_id, "/root/.ssh/")
        if exit_code != 0 :
            raise Exception(f"scp failed {result_stderr} and {result_stdout}")
    else:
        logger.info("/config/node-private-git-repo-key/node-private-git-repo-key not found, skipping")



    final_command = "./update-stack.sh"
    if FAKE_DEPLOYMENT:
        final_command = "echo \"This fake deployment was successful\""

    deploy_script= \
"""\
#!/bin/sh
### This script was generated by beekeeper ###
set -e
set -x

# check if k3s runs
kubectl get nodes

if [ ! -e "/opt/waggle-edge-stack" ] ; then
    cd /opt
    git clone https://github.com/waggle-sensor/waggle-edge-stack.git
fi

cd /opt/waggle-edge-stack
git pull origin main
#git checkout tags/v.1.0

cd /opt/waggle-edge-stack/kubernetes

""" + final_command


    #return {"result": "C"}
    try:
        node_ssh_with_logging(node_id, "cat > /tmp/deploy.sh", input_str=deploy_script)
        node_ssh_with_logging(node_id, "sh /tmp/deploy.sh")
    except Exception as e:
        raise Exception(f"node_ssh_with_logging failed: {str(e)}")

    try:
        register_wes_deployment_event(node_id, lock_tables=True, lock_requested_by="wes_deployment")
    except Exception as e:
        raise Exception(f"register_wes_deployment_event failed: {str(e)}")

    if this_debug:
        return {
            "waggle_ConfigMap": waggle_ConfigMap,
            "tls_secret":tls_secret,
            "upload_secret": upload_secret,
        }

        #node_creds["tls_result_stdout"] = result_stdout
        #node_creds["tls_result_stderr"] = result_stderr

    return {"success":True}
Пример #16
0
    def delete(self, beehive_id):

        bee_db = BeekeeperDB()
        result = bee_db.delete_object("beehives", "id", beehive_id )
        bee_db.close()
        return jsonify({"deleted": result})
Пример #17
0
    def get(self):
        """API to create keys, certificate and user for end-point.

        Arguments:
            id (str): unique ID for this end-point

        Returns:
            dict: end-point id, private key, public key and certificate
        """
        node_id = request.args.get("id", type=str)
        if not node_id:
            logger.debug("Registration failed: id missing")
            return f"Error: id missing\n", 500

        logger.debug("Register user [{}]".format(node_id))


        if DEFAULT_BEEHIVE:
            # first check the default beehive exists
            try:
                bee_db = BeekeeperDB()
                beehive_obj = bee_db.get_beehive(DEFAULT_BEEHIVE)
                bee_db.close()
            except Exception as e:
                raise Exception(f"error: get_beehive returned: {str(e)}")

            if not beehive_obj:
                raise Exception(f"error: Beehive {DEFAULT_BEEHIVE} is not known yet" )

        try:
            # create keypair and certificates for node (idempotent function)
            registration_result =  _register(node_id)
        except Exception as e:
            logger.debug(f"_register failed: {str(e)}")
            traceback.print_exc()
            logger.debug(f"Error: unable to register id [{node_id} , {str(e)}]")
            return f"Error: unable to register id [{node_id} , {str(e)}]\n", 500

        # update beekeeper db (create registartion event)
        try:
            register_node(node_id, lock_requested_by="register-resource")
        except Exception as e:
            logger.debug(f"Error: Creating registration event failed: {str(e)}")
            return f"Error: Creating registration event failed: {str(e)}", 500


        if DEFAULT_BEEHIVE:
            time.sleep(2) # see if that helps with the locks
            logger.debug("Adding user [{}] to default beehive".format(node_id))
            try:
                set_node_beehive(node_id, DEFAULT_BEEHIVE)
            except Exception as e:
                logger.debug(f"Error: Adding node to beehive {DEFAULT_BEEHIVE} failed: {str(e)}")
                # Do not let registration fail because of this
                return f"Error: Adding node to beehive {DEFAULT_BEEHIVE} failed: {str(e)}", 500
        else:
            logger.debug("No default beehive defined")


        logger.debug(f"success: responding with registration results")
        #return json.dumps(registration_result)
        return registration_result
Пример #18
0
def insert_log(postData, lock_tables=True, force=False, lock_requested_by="", replay=True):
    listData = None
    if isinstance( postData, dict ):
        listData = [ postData ]
        #print("Putting postData into array ", flush=True)
    else:
        listData = postData
        #print("Use postData as is ", flush=True)

    if not isinstance( listData, list ):
        raise Exception("list expected")





    logData = []
    default_effective_time = datetime.datetime.now(datetime.timezone.utc) # this way all operations in this submission have the exact same time
    for op in listData:

        for f in ["node_id", "operation", "field_name", "field_value", "source"]:
            if f not in op:
                raise Exception(f'Field {f} missing. Got: {json.dumps(op)}')

        if not force:
            if op["field_name"] == "beehive":
                raise Exception("Field \"beehive\" cannot be set via the /log resource")


        try:
            newLogDataEntry = {
                "node_id": op["node_id"],
                "table_name": "nodes_history" ,
                "operation": op["operation"],
                "field_name": op["field_name"],
                "new_value": op["field_value"],
                "source": op["source"],
                "effective_time" : op.get("effective_time", default_effective_time.isoformat()) }

        except Exception as ex:
            raise Exception(f"Unexpected error in creating newLogDataEntry : {ex}")

        logData.append(newLogDataEntry)
        #print("success", flush=True)

    bee_db = None
    try:
        bee_db = BeekeeperDB()
    except Exception as e:
        raise Exception(f"Could not create BeekeeperDB: {e}" )

    try:
        bee_db.nodes_log_add(logData, lock_tables=lock_tables, lock_requested_by=lock_requested_by, replay=replay)  #  effective_time=effective_time)
    except Exception as ex:
        raise Exception(f"nodes_log_add failed: {ex}" )




    bee_db.close()

    return