def update_project( project: schemas.ProjectUpdate, name: str, db_session: Session = Depends(deps.get_db_session), ): get_db().update_project(db_session, name, project.dict(exclude_unset=True)) return {}
def del_run( project: str, uid: str, tag: int = 0, db_session: Session = Depends(deps.get_db_session)): get_db().del_run(db_session, uid, project, iter=iter) return {}
def del_artifacts(project: str = "", name: str = "", tag: str = "", labels: List[str] = Query([], alias='label'), db_session: Session = Depends(deps.get_db_session)): get_db().del_artifacts(db_session, name, project, tag, labels) return {}
def del_artifact(project: str, uid: str, key: str, tag: str = "", db_session: Session = Depends(deps.get_db_session)): get_db().del_artifact(db_session, key, tag, project) return {}
def build_status(name: str = "", project: str = "", tag: str = "", offset: int = 0, logs: str = "on", db_session: Session = Depends(deps.get_db_session)): logs = strtobool(logs) fn = get_db().get_function(db_session, name, project, tag) if not fn: log_and_raise(HTTPStatus.NOT_FOUND, name=name, project=project, tag=tag) state = get_in(fn, "status.state", "") pod = get_in(fn, "status.build_pod", "") image = get_in(fn, "spec.build.image", "") out = b"" if not pod: if state == "ready": image = image or get_in(fn, "spec.image") return Response(content=out, media_type="text/plain", headers={ "function_status": state, "function_image": image, "builder_pod": pod }) logger.info("get pod {} status".format(pod)) state = get_k8s().get_pod_status(pod) logger.info("pod state={}".format(state)) if state == "succeeded": logger.info("build completed successfully") state = "ready" if state in ["failed", "error"]: logger.error("build {}, watch the build pod logs: {}".format( state, pod)) if logs and state != "pending": resp = get_k8s().logs(pod) if resp: out = resp.encode()[offset:] update_in(fn, "status.state", state) if state == "ready": update_in(fn, "spec.image", image) get_db().store_function(db_session, fn, name, project, tag) return Response(content=out, media_type="text/plain", headers={ "function_status": state, "function_image": image, "builder_pod": pod })
def del_runs( project: str = None, name: str = None, labels: List[str] = Query([], alias='label'), state: str = None, days_ago: int = 0, db_session: Session = Depends(deps.get_db_session)): get_db().del_runs(db_session, name, project, labels, state, days_ago) return {}
def _tag_objects(db_session, data, project, name): objs = [] for typ, query in data.items(): cls = table2cls(typ) if cls is None: err = f"unknown type - {typ}" log_and_raise(HTTPStatus.BAD_REQUEST, reason=err) # {"name": "bugs"} -> [Function.name=="bugs"] db_query = [getattr(cls, key) == value for key, value in query.items()] # TODO: Change _query to query? # TODO: Not happy about exposing db internals to API objs.extend(db_session.query(cls).filter(*db_query)) get_db().tag_objects(db_session, objs, project, name) return objs
async def store_artifact( request: Request, project: str, uid: str, key: str, tag: str = "", iter: int = 0, db_session: Session = Depends(deps.get_db_session), ): data = None try: data = await request.json() except ValueError: log_and_raise(HTTPStatus.BAD_REQUEST, reason="bad JSON body") logger.debug(data) await run_in_threadpool( get_db().store_artifact, db_session, key, data, uid, iter=iter, tag=tag, project=project, ) return {}
def list_runs( project: str = None, name: str = None, uid: str = None, labels: List[str] = Query([], alias='label'), state: str = None, last: int = 0, sort: str = "on", iter: str = "on", db_session: Session = Depends(deps.get_db_session), ): sort = strtobool(sort) iter = strtobool(iter) runs = get_db().list_runs( db_session, name=name, uid=uid, project=project, labels=labels, state=state, sort=sort, last=last, iter=iter, ) return { "runs": runs, }
async def store_function( request: Request, project: str, name: str, tag: str = "", versioned: bool = False, db_session: Session = Depends(deps.get_db_session), ): data = None try: data = await request.json() except ValueError: log_and_raise(HTTPStatus.BAD_REQUEST, reason="bad JSON body") logger.debug(data) logger.info("store function: project=%s, name=%s, tag=%s", project, name, tag) hash_key = await run_in_threadpool( get_db().store_function, db_session, data, name, project, tag=tag, versioned=versioned, ) return { 'hash_key': hash_key, }
def add_project(project: schemas.ProjectCreate, db_session: Session = Depends(deps.get_db_session)): project_id = get_db().add_project(db_session, project.dict()) return { "id": project_id, "name": project.name, }
def _start_function(db_session, data): logger.info("start_function:\n{}".format(data)) url = data.get("functionUrl") if not url: log_and_raise(HTTPStatus.BAD_REQUEST, reason="runtime error: functionUrl not specified") project, name, tag, hash_key = parse_function_uri(url) runtime = get_db().get_function(db_session, name, project, tag, hash_key) if not runtime: log_and_raise( HTTPStatus.BAD_REQUEST, reason="runtime error: function {} not found".format(url)) fn = new_function(runtime=runtime) resource = runtime_resources_map.get(fn.kind) if "start" not in resource: log_and_raise( HTTPStatus.BAD_REQUEST, reason="runtime error: 'start' not supported by this runtime") try: run_db = get_run_db_instance(db_session) fn.set_db_connection(run_db) # resp = resource["start"](fn) # TODO: handle resp? resource["start"](fn) fn.save(versioned=False) logger.info("Fn:\n %s", fn.to_yaml()) except Exception as err: logger.error(traceback.format_exc()) log_and_raise(HTTPStatus.BAD_REQUEST, reason="runtime error: {}".format(err)) return fn
def list_tags(project: str, db_session: Session = Depends(deps.get_db_session)): tags = get_db().list_tags(db_session, project) return { "project": project, "tags": tags, }
def get_run_db_instance(db_session: Session): db = get_db() if isinstance(db, SQLDB): run_db = SQLRunDB(db.dsn, db_session, db.get_projects_cache()) else: run_db = db.db run_db.connect() return run_db
def delete_runtimes(label_selector: str = None, force: bool = False, db_session: Session = Depends(deps.get_db_session)): for kind in RuntimeKinds.runtime_with_handlers(): runtime_handler = get_runtime_handler(kind) runtime_handler.delete_resources(get_db(), db_session, label_selector, force) return Response(status_code=status.HTTP_204_NO_CONTENT)
def list_functions(project: str = config.default_project, name: str = None, tag: str = None, labels: List[str] = Query([], alias='label'), db_session: Session = Depends(deps.get_db_session)): funcs = get_db().list_functions(db_session, name, project, tag, labels) return { "funcs": list(funcs), }
def del_tag(project: str, name: str, db_session: Session = Depends(deps.get_db_session)): count = get_db().del_tag(db_session, project, name) return { "project": project, "name": name, "count": count, }
def get_function(project: str, name: str, tag: str = "", hash_key="", db_session: Session = Depends(deps.get_db_session)): func = get_db().get_function(db_session, name, project, tag, hash_key) return { "func": func, }
def _cleanup_runtimes(): logger.debug('Cleaning runtimes') db_session = create_session() try: for kind in RuntimeKinds.runtime_with_handlers(): runtime_handler = get_runtime_handler(kind) runtime_handler.delete_resources(get_db(), db_session) finally: close_session(db_session)
def read_run( project: str, uid: str, iter: int = 0, db_session: Session = Depends(deps.get_db_session)): data = get_db().read_run(db_session, uid, project, iter=iter) return { "data": data, }
def get_tagged(project: str, name: str, db_session: Session = Depends(deps.get_db_session)): objs = get_db().find_tagged(db_session, project, name) return { "project": project, "tag": name, "objects": [db2dict(obj) for obj in objs], }
def list_artifacts(project: str = config.default_project, name: str = None, tag: str = None, labels: List[str] = Query([], alias='label'), db_session: Session = Depends(deps.get_db_session)): artifacts = get_db().list_artifacts(db_session, name, project, tag, labels) return { "artifacts": artifacts, }
def get_project(name: str, db_session: Session = Depends(deps.get_db_session)): project = get_db().get_project(db_session, name) if not project: log_and_raise(error=f"project {name!r} not found") project.users = [u.name for u in project.users] return { "project": project, }
def _reschedule_tasks(): db_session = None try: db_session = create_session() for data in get_db().list_schedules(db_session): if "schedule" not in data: logger.warning("bad scheduler data - %s", data) continue submit(db_session, data) finally: close_session(db_session)
def read_artifact( project: str, key: str, tag: str = "latest", iter: int = 0, db_session: Session = Depends(deps.get_db_session), ): data = get_db().read_artifact(db_session, key, tag=tag, iter=iter, project=project) return { "data": data, }
def get_log( db_session: Session, project: str, uid: str, size: int = -1, offset: int = 0, source: LogSources = LogSources.AUTO, ): out = b"" log_file = log_path(project, uid) status = None if log_file.exists() and source in [ LogSources.AUTO, LogSources.PERSISTENCY ]: with log_file.open("rb") as fp: fp.seek(offset) out = fp.read(size) status = "" elif source in [LogSources.AUTO, LogSources.K8S]: data = get_db().read_run(db_session, uid, project) if not data: log_and_raise(HTTPStatus.NOT_FOUND, project=project, uid=uid) status = get_in(data, "status.state", "") if get_k8s(): pods = get_k8s().get_logger_pods(uid) if pods: pod, new_status = list(pods.items())[0] new_status = new_status.lower() # TODO: handle in cron/tracking if new_status != "pending": resp = get_k8s().logs(pod) if resp: out = resp.encode()[offset:] if status == "running": now = now_date().isoformat() update_in(data, "status.last_update", now) if new_status == "failed": update_in(data, "status.state", "error") update_in(data, "status.error", "error, check logs") get_db().store_run(db_session, data, uid, project) if new_status == "succeeded": update_in(data, "status.state", "completed") get_db().store_run(db_session, data, uid, project) status = new_status elif status == "running": update_in(data, "status.state", "error") update_in(data, "status.error", "pod not found, maybe terminated") get_db().store_run(db_session, data, uid, project) status = "failed" return out, status
def delete_runtime(kind: str, label_selector: str = None, force: bool = False, db_session: Session = Depends(deps.get_db_session)): if kind not in RuntimeKinds.runtime_with_handlers(): log_and_raise(status.HTTP_400_BAD_REQUEST, kind=kind, err='Invalid runtime kind') runtime_handler = get_runtime_handler(kind) runtime_handler.delete_resources(get_db(), db_session, label_selector, force) return Response(status_code=status.HTTP_204_NO_CONTENT)
async def update_run( request: Request, project: str, uid: str, iter: int = 0, db_session: Session = Depends(deps.get_db_session)): data = None try: data = await request.json() except ValueError: log_and_raise(HTTPStatus.BAD_REQUEST, reason="bad JSON body") logger.debug(data) await run_in_threadpool(get_db().update_run, db_session, data, uid, project, iter=iter) logger.info("update run: {}".format(data)) return {}
def list_projects(full: str = "no", db_session: Session = Depends(deps.get_db_session)): full = strtobool(full) fn = db2dict if full else attrgetter("name") projects = [] for p in get_db().list_projects(db_session): if isinstance(p, dict): if full: projects.append(p) else: projects.append(p.get("name")) else: projects.append(fn(p)) return { "projects": projects, }
def list_schedules(db_session: Session = Depends(deps.get_db_session)): schedules = get_db().list_schedules(db_session) return { "schedules": list(schedules), }