def get(self, node_id): try: bee_db = BeekeeperDB() results = bee_db.get_node_keypair(node_id) bee_db.close() except Exception as e: raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if not results: raise ErrorResponse(f"Not found." , status_code=HTTPStatus.NOT_FOUND ) return jsonify(results)
def get(self, node_id): try: bee_db = BeekeeperDB() node_state = bee_db.get_node_state(node_id) bee_db.close() except Exception as e: raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if node_state == None: raise ErrorResponse(f"Error: node {node_id} not found") return { "data" : node_state }
def wrapper2(self, **kwargs): authenticated = request.environ['authenticated'] if not authenticated: raise ErrorResponse('Not authenticated', status_code=HTTPStatus.UNAUTHORIZED) return func(self, **kwargs)
def get(self): try: bee_db = BeekeeperDB() node_state = bee_db.list_latest_state() bee_db.close() except Exception as e: raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) return { "data" : node_state }
def get(self, app_id): try: js = jenkins_server.JenkinsServer(host=config.jenkins_server, username=config.jenkins_user, password=config.jenkins_token) except Exception as e: raise ErrorResponse( f'JenkinsServer({config.jenkins_server}, {config.jenkins_user}) returned: {str(e)}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) source_name = request.values.get("source", "default") #source_name = "default" # strategy to find last build # 1. check db field "number" # 2. use global queue id to map to app-specific build number # 3. take whatever is reported as last build (IS MISLEADING, returns previous build) ecr_db = ecrdb.EcrDB() app_spec = ecr_db.getApp(app_id) app_human_id = createJenkinsName(app_spec, source_name) # strategy 1: try to find build number in database number = -1 architectures = "" try: number, architectures = ecr_db.getBuildInfo(app_id, source_name) except Exception as e: raise ErrorResponse(f'Could not get build number: {str(e)}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if number != -1: # get Jenkins build info try: buildInfo = js.server.get_build_info(app_human_id, number) except Exception as e: raise Exception(f'js.server.get_build_info returned: {str(e)}') return buildInfo return {"error": f"number is negative"}
def post(self): try: postData = request.get_json(force=True, silent=False) except Exception as e: raise ErrorResponse(f"Error parsing json: { sys.exc_info()[0] } {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if not postData: raise ErrorResponse(f"Could not parse json." , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) try: insert_log(postData, lock_requested_by="log-resource", replay=True) except Exception as e: raise ErrorResponse(f"Could not insert log: {str(e)}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) response = {"success" : 1} return response
def wrapper(self, app_id): authenticated = request.environ['authenticated'] ecr_db = ecrdb.EcrDB() if not authenticated: if not (ecr_db.hasPermission(app_id, "GROUP", "AllUsers" , permissions)): raise ErrorResponse(f'Not authorized.', status_code=HTTPStatus.UNAUTHORIZED) requestUser = request.environ.get('user', "") isAdmin = request.environ.get('admin', False) if (isAdmin or ecr_db.hasPermission(app_id, "USER", requestUser ,permissions)): return func(self, app_id) raise ErrorResponse(f'Not authorized.', status_code=HTTPStatus.UNAUTHORIZED)
def delete(self, app_id): ecr_db = ecrdb.EcrDB() try: ecr_db.deleteApp(app_id) except Exception as e: raise ErrorResponse(f'Error deleting app: {e}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) return {"deleted": 1}
def delete(self, node_id): try: bee_db = BeekeeperDB() result_count = bee_db.delete_object( "node_credentials", "id", node_id) bee_db.close() except Exception as e: raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) return jsonify({"deleted": result_count})
def post(self): try: #request.get postData = request.get_json(force=True, silent=False) except Exception as e: raise ErrorResponse(f"Error parsing json: { sys.exc_info()[0] } {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if not "id" in postData: raise ErrorResponse(f"Field id is missing" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if not "key-type" in postData: raise ErrorResponse(f"Field key-type is missing" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) beehive_id = postData["id"] key_type = postData["key-type"] #key_type_args = postData.get("key-type-args", "") bee_db = BeekeeperDB() #TODO check if beehive already exists beehive_obj = {"id": beehive_id} #modified = 0 #updates = {} for key in ["key_type", "key_type_args", "rmq_host", "rmq_port" , "upload_host" , "upload_port"]: key_dash = key.replace("_", "-") if not key_dash in postData: continue beehive_obj[key] = postData[key_dash] modified = bee_db.insert_object("beehives", beehive_obj, force=True) bee_db.close() return jsonify({"modified": modified})
def post(self, node_id): try: #request.get postData = request.get_json(force=True, silent=False) except Exception as e: raise ErrorResponse(f"Error parsing json: { sys.exc_info()[0] } {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if not postData: raise ErrorResponse(f"Could not parse json." , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) valid_keys = {"ssh_key_private", "ssh_key_public"} expected_keys = valid_keys for key in postData: if key not in valid_keys: raise ErrorResponse(f"Key {key} not supported" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) for key in expected_keys: if key not in postData: raise ErrorResponse(f"Key {key} missing" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) try: bee_db = BeekeeperDB() results = bee_db.set_node_keypair(node_id, postData) bee_db.close() except Exception as e: raise ErrorResponse(f"Unexpected error: {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) return "success"
def post(self, beehive_id): expected_forms = ["tls-key", "tls-cert", "ssh-key", "ssh-pub", "ssh-cert"] count_updated = 0 data={} try: bee_db = BeekeeperDB() obj = bee_db.get_beehive(beehive_id) if not obj: raise Exception(f"Beehive {beehive_id} not found" ) for formname in request.files: if not formname in expected_forms: raise Exception(f"Formname {formname} not supported" ) # we could remove this check... for formname in expected_forms: if not formname in request.files: raise Exception(f"Formname {formname} missing" ) for formname in request.files: formdata = request.files.get(formname).read().decode("utf-8") if not formdata: raise Exception(f"Field {formname} empty" ) data[formname] = formdata #logger.debug(f"data: {formname} {data[formname]}") #filename = secure_filename(file.filename) #logger.debug(f"filename: {filename}") for formname in data: col_name = formname.replace("-", "_ca_") count_updated += bee_db.update_object_field("beehives", col_name, data[formname], "id", beehive_id) bee_db.close() except Exception as e: raise ErrorResponse(f"something failed: {str(e)}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) return jsonify({"modified": count_updated})
def post(self, node_id): try: #request.get postData = request.get_json(force=True, silent=False) except Exception as e: raise ErrorResponse(f"Error parsing json: { sys.exc_info()[0] } {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) this_debug = request.args.get('debug', "false") in ["true", "1"] force = request.args.get('force', "false") in ["true", "1"] if "assign_beehive" in postData: assign_beehive = postData["assign_beehive"] try: set_node_beehive(node_id, assign_beehive) except Exception as e: logger.error(e) raise ErrorResponse(f"set_node_beehive returned: { type(e).__name__ }: {str(e)} {ShowException()}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if "deploy_wes" in postData: try: result = deploy_wes(node_id, this_debug, force=force) except Exception as e: logger.error(e) raise ErrorResponse(f"deploy_wes returned: { type(e).__name__ }: {str(e)} {ShowException()}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) return jsonify(result) return jsonify({"success":True})
def get(self): view = request.args.get('view', "") try: bee_db = BeekeeperDB() fields = ["id"] if view == "full": fields=None result = bee_db.get_objects('beehives', fields = fields) bee_db.close() except Exception as e: raise ErrorResponse(f"Error getting list of beehives: { type(e).__name__ } {e}" , status_code=HTTPStatus.INTERNAL_SERVER_ERROR) return jsonify({"data":result})
def get(self, node_id): try: bee_db = BeekeeperDB() node_state = bee_db.get_node_state(node_id) except Exception as e: raise ErrorResponse(f"Unexpected error: {e}", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if "timestamp" in node_state: node_state["timestamp"] = node_state["timestamp"].isoformat() if "registration_event" in node_state: node_state["registration_event"] = node_state[ "registration_event"].isoformat() return {"data": node_state}
def put(self, app_id): # example to make app public: # curl -X PUT localhost:5000/permissions/{id} -H "Authorization: sage user:testuser" -d '{"granteeType": "GROUP", "grantee": "AllUsers", "permission": "READ"}' postData = request.get_json(force=True) for key in ["granteeType", "grantee", "permission"]: if not key in postData: raise ErrorResponse( f'Field {key} missing', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) ecr_db = ecrdb.EcrDB() result = ecr_db.addPermission(app_id, postData["granteeType"], postData["grantee"], postData["permission"]) obj = {"added": result} return jsonify(obj)
def wrapper2(self, namespace=None, repository=None, version=None): authenticated = request.environ['authenticated'] resourceType = "" resourceName = "" if repository: if not namespace: raise ErrorResponse( f'namespace missing (should not happen, code:a)', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) resourceType = "repository" resourceName = f'{namespace}/{repository}' elif namespace: resourceType = "namespace" resourceName = namespace else: raise ErrorResponse( f'namespace missing (should not happen, code:b)', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) ecr_db = ecrdb.EcrDB() if not authenticated: if (ecr_db.hasPermission(resourceType, resourceName, "GROUP", "AllUsers", permission)): return func(self, namespace, repository, version) raise ErrorResponse(f'Not authorized.', status_code=HTTPStatus.UNAUTHORIZED) requestUser = request.environ.get('user', "") isAdmin = request.environ.get('admin', False) if not repository: # check namespace permission only if (isAdmin or ecr_db.hasPermission(resourceType, resourceName, "USER", requestUser, permission)): #return func(self, namespace, repository, version) return func(self, namespace) raise ErrorResponse( f'Not authorized. (User {requestUser} does not have permission {permission} for {resourceType} {resourceName})', status_code=HTTPStatus.UNAUTHORIZED) # # check repository permission and namespace-inherited permission hasNamespaceAccess = ecr_db.hasPermission("namespace", namespace, "USER", requestUser, permission) if (isAdmin or hasNamespaceAccess or ecr_db.hasPermission(resourceType, resourceName, "USER", requestUser, permission)): return func(self, namespace, repository, version) raise ErrorResponse( f'Not authorized. (User {requestUser} does not have permission {permission} for {resourceType} {resourceName})', status_code=HTTPStatus.UNAUTHORIZED)
def post(self, app_id): host = config.jenkins_server username = config.jenkins_user password = config.jenkins_token try: js = jenkins_server.JenkinsServer(host, username, password) except Exception as e: raise ErrorResponse( f'JenkinsServer({host}, {username}) returned: {str(e)}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) ecr_db = ecrdb.EcrDB() app_spec = ecr_db.getApp(app_id) source_name = request.values.get("source", "default") #source_name = "default" sources = app_spec.get("sources", []) source = None for src in sources: src_name = src.get("name", "none") if src_name == source_name: source = src break if not source: raise ErrorResponse(f'No source found in app spec', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) app_human_id = createJenkinsName(app_spec, source_name) overwrite = False if js.hasJenkinsJob(app_human_id): overwrite = True try: js.createJob(app_human_id, app_spec, source_name, overwrite=overwrite) except Exception as e: raise ErrorResponse(f'createJob() returned: {str(e)}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) queue_item_number = js.build_job(app_human_id) queue_item = None # this loop will waitb for Jenkins to return build number # note that the previous queue_item_number is a temporary global queue number which will be useless after some time. number = -1 while number == -1: time.sleep(2) try: queue_item = js.server.get_queue_item(queue_item_number) except Exception as e: # pragma: no cover if not "does not exist" in str(e): raise ErrorResponse( f'get_queue_item() returned: {str(e)}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if not queue_item: continue executable = queue_item.get("executable", None) if not executable: continue number = executable.get("number", None) if not number: # pragma: no cover continue break architectures = source.get("architectures", None) if not architectures: raise ErrorResponse(f'architectures not specified in source', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) try: ecr_db.SaveBuildInfo(app_id, source_name, number, architectures) except Exception as e: raise ErrorResponse( f'error inserting build info for {app_id}, {source_name}, {number} , SaveBuildInfo: {str(e)}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) #time.sleep(6) #queued_item = js.server.get_queue_item(queue_item_number) #returnObj = {"queue_item_number":queue_item_number, "queued_item": queued_item} #return returnObj build_request_counter.inc(1) return {"build_number": number}
def post(self): try: #request.get postData = request.get_json(force=True, silent=False) except Exception as e: raise ErrorResponse( f"Error parsing json: { sys.exc_info()[0] } {e}", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if not postData: raise ErrorResponse(f"Could not parse json.", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) listData = None if isinstance(postData, dict): listData = [postData] print("Putting postData into array ", flush=True) else: listData = postData print("Use postData as is ", flush=True) if not isinstance(listData, list): raise ErrorResponse("list expected", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) bee_db = None try: bee_db = BeekeeperDB() except Exception as e: raise ErrorResponse(f"Could not create BeekeeperDB: {e}", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) logData = [] default_effective_time = datetime.datetime.now( datetime.timezone.utc ) # this way all operations in this submission have the exact same time for op in listData: for f in [ "node_id", "operation", "field_name", "field_value", "source" ]: if f not in op: raise Exception( f'Field {f} missing. Got: {json.dumps(op)}') try: newLogDataEntry = { "node_id": op["node_id"], "table_name": "nodes_log", "operation": op["operation"], "field_name": op["field_name"], "new_value": op["field_value"], "source": op["source"], "effective_time": op.get("effective_time", default_effective_time.isoformat()) } except Exception as ex: raise ErrorResponse( f"Unexpected error in creating newLogDataEntry : {ex}", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) logData.append(newLogDataEntry) #print("success", flush=True) try: bee_db.nodes_log_add(logData) # effective_time=effective_time) except Exception as ex: raise ErrorResponse(f"nodes_log_add failed: {ex}", status_code=HTTPStatus.INTERNAL_SERVER_ERROR) response = {"success": 1} return response
def post(self): # example # curl -X POST localhost:5000/apps -d '{"name" : "testapp", "description": "blabla", "version" : "1.0", "source" :"https://github.com/user/repo.git#v1.0", "inputs": [{"id":"speed" , "type":"int" }] , "metadata": {"my-science-data" : 12345} }' # TODO authentication # TODO set owner requestUser = request.environ.get('user', "") postData = request.get_json(force=True) for key in postData: if not key in config.valid_fields_set: #return {"error": f'Field {key} not supported'} raise ErrorResponse( f'Field {key} not supported', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) # if required for key in config.required_fields: if not key in postData: #return {"error": f'Required field {key} is missing'} raise ErrorResponse( f'Required field {key} is missing', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) value = postData[key] if len(value) == 0: #return {"error": f'Required field {key} is missing'} raise ErrorResponse( f'Required field {key} is missing', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) ##### name appName = postData.get("name", "") appNameArray = appName.split("/", 2) if len(appNameArray) > 1: raise ErrorResponse(f'Name should not contain a slash', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) vc = '[.a-zA-Z0-9_-]' p = re.compile(f'[a-zA-Z0-9_]{vc}+', re.ASCII) if not p.match(appName): #return {"error": f'Name can only consist of [0-9a-zA-Z-_.] characters and only start with [0-9a-zA-Z] characters.'} raise ErrorResponse( f'Name can only consist of [0-9a-zA-Z-_.] characters and only start with [0-9a-zA-Z] characters.', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) ##### source # source # [email protected]:<user>/<repo>.git#<tag> # https://github.com/<user>/<repo>.git#<tag> # http://sagecontinuum.org/bucket/<bucket_id> sourcesArray = postData.get("sources", []) if len(sourcesArray) == 0: raise ErrorResponse("Field source is missing") #source_public_git_pattern = re.compile(f'https://github.com/{vc}+/{vc}+.git#{vc}+') #source_private_git_pattern = re.compile(f'[email protected]/{vc}+/{vc}+.git#{vc}+') #source_sage_store_pattern = re.compile(f'http://sagecontinuum.org/bucket/[0-9a-z.]+') #source_matched = False #for p in [source_public_git_pattern, source_private_git_pattern , source_sage_store_pattern]: # if p.match(appSource): # source_matched = True # break #if not source_matched: # raise ErrorResponse('Could not parse source field', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) ##### inputs # inputs validation appInputs = postData.get("inputs", []) if len(appInputs) > 0: for app_input in appInputs: for field in app_input: if not field in config.input_fields_valid: raise ErrorResponse( f'Input field {field} not supported', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) for expected in config.input_fields_valid: if not expected in app_input: raise ErrorResponse( f'Expected field {expected} missing', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) input_type = app_input["type"] if not input_type in config.input_valid_types: raise ErrorResponse( f'Input type {input_type} not supported', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) appInputs_str = json.dumps(appInputs) ##### resources #resources_str = None resourcesArray = postData.get("resources", []) if not isinstance(resourcesArray, list): raise ErrorResponse(f'Field resources has to be an array', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) #resources_str = json.dumps(resourcesArray) ##### metadata appMetadata = postData.get("metadata", None) ##### create dbObject dbObject = {} for key in config.valid_fields_set: dbObject[key] = "" if appMetadata: #raise ErrorResponse(f'metadata is missing', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) if not isinstance(appMetadata, dict): raise ErrorResponse( f'Field metadata has to be an object, got {str(appMetadata)}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) dbObject["metadata"] = json.dumps(appMetadata) dbObject["name"] = appName dbObject["inputs"] = appInputs_str #copy fields for key in ["description", "version", "namespace"]: dbObject[key] = postData[key] dbObject["owner"] = requestUser # create INSERT statment dynamically values = [] variables = [] for key in config.dbFields: values.append(dbObject[key]) variables.append("%s") variables_str = ",".join(variables) newID = uuid.uuid4() newID_str = str(newID) ecr_db = ecrdb.EcrDB() for build_source in sourcesArray: source_name = build_source.get("name", "default") architectures_array = build_source.get("architectures", []) if len(architectures_array) == 0: raise ErrorResponse("architectures missing in source") ##### architecture for arch in architectures_array: if not arch in config.architecture_valid: valid_arch_str = ",".join(config.architecture_valid) raise ErrorResponse( f'Architecture {arch} not supported, valid values: {valid_arch_str}', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) architectures = json.dumps(architectures_array) url = build_source.get("url", "") if url == "": raise ErrorResponse("url missing in source") branch = build_source.get("branch", "master") if branch == "": raise ErrorResponse("branch missing in source") directory = build_source.get("directory", ".") if directory == "": directory = "." dockerfile = build_source.get("dockerfile", "Dockerfile") if dockerfile == "": dockerfile = "Dockerfile" build_args_dict = build_source.get("build_args", {}) if not isinstance(build_args_dict, dict): raise ErrorResponse( f'build_args needs to be a dictonary', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) for key in build_args_dict: value = build_args_dict[key] if not isinstance(value, str): raise ErrorResponse( f'build_args values have to be strings', status_code=HTTPStatus.INTERNAL_SERVER_ERROR) build_args_str = json.dumps(build_args_dict) #appSources = postData["sources"] stmt = f'INSERT INTO Sources ( id, name, architectures , url, branch, directory, dockerfile, build_args ) VALUES (UUID_TO_BIN(%s) , %s , %s, %s, %s, %s, %s, %s)' print(f"insert statement: {stmt}") print(f"build_args_str: {build_args_str}") ecr_db.cur.execute(stmt, (newID_str, source_name, architectures, url, branch, directory, dockerfile, build_args_str)) for res in resourcesArray: res_str = json.dumps(res) stmt = f'INSERT INTO Resources ( id, resource) VALUES (UUID_TO_BIN(%s) , %s)' ecr_db.cur.execute(stmt, ( newID_str, res_str, )) stmt = f'INSERT INTO Apps ( id, {config.dbFields_str}) VALUES (UUID_TO_BIN(%s) ,{variables_str})' print(f'stmt: {stmt}', file=sys.stderr) ecr_db.cur.execute(stmt, (newID_str, *values)) stmt = f'INSERT INTO AppPermissions ( id, granteeType , grantee, permission) VALUES (UUID_TO_BIN(%s) , %s, %s, %s)' ecr_db.cur.execute(stmt, (newID_str, "USER", requestUser, "FULL_CONTROL")) ecr_db.db.commit() #print(f'row: {row}', file=sys.stderr) #dbObject["id"] = newID #content = {} #content["data"] = dbObject returnObj = ecr_db.getApp(newID_str) app_submission_counter.inc(1) #args = parser.parse_args() return returnObj