def update_project(
project: schemas.ProjectUpdate,
name: str,
db_session: Session = Depends(deps.get_db_session),
):
get_db().update_project(db_session, name, project.dict(exclude_unset=True))
return {}
def del_run(
project: str,
uid: str,
tag: int = 0,
db_session: Session = Depends(deps.get_db_session)):
get_db().del_run(db_session, uid, project, iter=iter)
return {}
def del_artifacts(project: str = "",
name: str = "",
tag: str = "",
labels: List[str] = Query([], alias='label'),
db_session: Session = Depends(deps.get_db_session)):
get_db().del_artifacts(db_session, name, project, tag, labels)
return {}
def del_artifact(project: str,
uid: str,
key: str,
tag: str = "",
db_session: Session = Depends(deps.get_db_session)):
get_db().del_artifact(db_session, key, tag, project)
return {}
def build_status(name: str = "",
project: str = "",
tag: str = "",
offset: int = 0,
logs: str = "on",
db_session: Session = Depends(deps.get_db_session)):
logs = strtobool(logs)
fn = get_db().get_function(db_session, name, project, tag)
if not fn:
log_and_raise(HTTPStatus.NOT_FOUND,
name=name,
project=project,
tag=tag)
state = get_in(fn, "status.state", "")
pod = get_in(fn, "status.build_pod", "")
image = get_in(fn, "spec.build.image", "")
out = b""
if not pod:
if state == "ready":
image = image or get_in(fn, "spec.image")
return Response(content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod
})
logger.info("get pod {} status".format(pod))
state = get_k8s().get_pod_status(pod)
logger.info("pod state={}".format(state))
if state == "succeeded":
logger.info("build completed successfully")
state = "ready"
if state in ["failed", "error"]:
logger.error("build {}, watch the build pod logs: {}".format(
state, pod))
if logs and state != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
update_in(fn, "status.state", state)
if state == "ready":
update_in(fn, "spec.image", image)
get_db().store_function(db_session, fn, name, project, tag)
return Response(content=out,
media_type="text/plain",
headers={
"function_status": state,
"function_image": image,
"builder_pod": pod
})
def del_runs(
project: str = None,
name: str = None,
labels: List[str] = Query([], alias='label'),
state: str = None,
days_ago: int = 0,
db_session: Session = Depends(deps.get_db_session)):
get_db().del_runs(db_session, name, project, labels, state, days_ago)
return {}
def _tag_objects(db_session, data, project, name):
objs = []
for typ, query in data.items():
cls = table2cls(typ)
if cls is None:
err = f"unknown type - {typ}"
log_and_raise(HTTPStatus.BAD_REQUEST, reason=err)
# {"name": "bugs"} -> [Function.name=="bugs"]
db_query = [getattr(cls, key) == value for key, value in query.items()]
# TODO: Change _query to query?
# TODO: Not happy about exposing db internals to API
objs.extend(db_session.query(cls).filter(*db_query))
get_db().tag_objects(db_session, objs, project, name)
return objs
async def store_artifact(
request: Request,
project: str,
uid: str,
key: str,
tag: str = "",
iter: int = 0,
db_session: Session = Depends(deps.get_db_session),
):
data = None
try:
data = await request.json()
except ValueError:
log_and_raise(HTTPStatus.BAD_REQUEST, reason="bad JSON body")
logger.debug(data)
await run_in_threadpool(
get_db().store_artifact,
db_session,
key,
data,
uid,
iter=iter,
tag=tag,
project=project,
)
return {}
def list_runs(
project: str = None,
name: str = None,
uid: str = None,
labels: List[str] = Query([], alias='label'),
state: str = None,
last: int = 0,
sort: str = "on",
iter: str = "on",
db_session: Session = Depends(deps.get_db_session),
):
sort = strtobool(sort)
iter = strtobool(iter)
runs = get_db().list_runs(
db_session,
name=name,
uid=uid,
project=project,
labels=labels,
state=state,
sort=sort,
last=last,
iter=iter,
)
return {
"runs": runs,
}
async def store_function(
request: Request,
project: str,
name: str,
tag: str = "",
versioned: bool = False,
db_session: Session = Depends(deps.get_db_session),
):
data = None
try:
data = await request.json()
except ValueError:
log_and_raise(HTTPStatus.BAD_REQUEST, reason="bad JSON body")
logger.debug(data)
logger.info("store function: project=%s, name=%s, tag=%s", project, name,
tag)
hash_key = await run_in_threadpool(
get_db().store_function,
db_session,
data,
name,
project,
tag=tag,
versioned=versioned,
)
return {
'hash_key': hash_key,
}
def add_project(project: schemas.ProjectCreate,
db_session: Session = Depends(deps.get_db_session)):
project_id = get_db().add_project(db_session, project.dict())
return {
"id": project_id,
"name": project.name,
}
def _start_function(db_session, data):
logger.info("start_function:\n{}".format(data))
url = data.get("functionUrl")
if not url:
log_and_raise(HTTPStatus.BAD_REQUEST,
reason="runtime error: functionUrl not specified")
project, name, tag, hash_key = parse_function_uri(url)
runtime = get_db().get_function(db_session, name, project, tag, hash_key)
if not runtime:
log_and_raise(
HTTPStatus.BAD_REQUEST,
reason="runtime error: function {} not found".format(url))
fn = new_function(runtime=runtime)
resource = runtime_resources_map.get(fn.kind)
if "start" not in resource:
log_and_raise(
HTTPStatus.BAD_REQUEST,
reason="runtime error: 'start' not supported by this runtime")
try:
run_db = get_run_db_instance(db_session)
fn.set_db_connection(run_db)
# resp = resource["start"](fn) # TODO: handle resp?
resource["start"](fn)
fn.save(versioned=False)
logger.info("Fn:\n %s", fn.to_yaml())
except Exception as err:
logger.error(traceback.format_exc())
log_and_raise(HTTPStatus.BAD_REQUEST,
reason="runtime error: {}".format(err))
return fn
def list_tags(project: str,
db_session: Session = Depends(deps.get_db_session)):
tags = get_db().list_tags(db_session, project)
return {
"project": project,
"tags": tags,
}
def get_run_db_instance(db_session: Session):
db = get_db()
if isinstance(db, SQLDB):
run_db = SQLRunDB(db.dsn, db_session, db.get_projects_cache())
else:
run_db = db.db
run_db.connect()
return run_db
def delete_runtimes(label_selector: str = None,
force: bool = False,
db_session: Session = Depends(deps.get_db_session)):
for kind in RuntimeKinds.runtime_with_handlers():
runtime_handler = get_runtime_handler(kind)
runtime_handler.delete_resources(get_db(), db_session, label_selector,
force)
return Response(status_code=status.HTTP_204_NO_CONTENT)
def list_functions(project: str = config.default_project,
name: str = None,
tag: str = None,
labels: List[str] = Query([], alias='label'),
db_session: Session = Depends(deps.get_db_session)):
funcs = get_db().list_functions(db_session, name, project, tag, labels)
return {
"funcs": list(funcs),
}
def del_tag(project: str,
name: str,
db_session: Session = Depends(deps.get_db_session)):
count = get_db().del_tag(db_session, project, name)
return {
"project": project,
"name": name,
"count": count,
}
def get_function(project: str,
name: str,
tag: str = "",
hash_key="",
db_session: Session = Depends(deps.get_db_session)):
func = get_db().get_function(db_session, name, project, tag, hash_key)
return {
"func": func,
}
def _cleanup_runtimes():
logger.debug('Cleaning runtimes')
db_session = create_session()
try:
for kind in RuntimeKinds.runtime_with_handlers():
runtime_handler = get_runtime_handler(kind)
runtime_handler.delete_resources(get_db(), db_session)
finally:
close_session(db_session)
def read_run(
project: str,
uid: str,
iter: int = 0,
db_session: Session = Depends(deps.get_db_session)):
data = get_db().read_run(db_session, uid, project, iter=iter)
return {
"data": data,
}
def get_tagged(project: str,
name: str,
db_session: Session = Depends(deps.get_db_session)):
objs = get_db().find_tagged(db_session, project, name)
return {
"project": project,
"tag": name,
"objects": [db2dict(obj) for obj in objs],
}
def list_artifacts(project: str = config.default_project,
name: str = None,
tag: str = None,
labels: List[str] = Query([], alias='label'),
db_session: Session = Depends(deps.get_db_session)):
artifacts = get_db().list_artifacts(db_session, name, project, tag, labels)
return {
"artifacts": artifacts,
}
def get_project(name: str, db_session: Session = Depends(deps.get_db_session)):
project = get_db().get_project(db_session, name)
if not project:
log_and_raise(error=f"project {name!r} not found")
project.users = [u.name for u in project.users]
return {
"project": project,
}
def _reschedule_tasks():
db_session = None
try:
db_session = create_session()
for data in get_db().list_schedules(db_session):
if "schedule" not in data:
logger.warning("bad scheduler data - %s", data)
continue
submit(db_session, data)
finally:
close_session(db_session)
def read_artifact(
project: str,
key: str,
tag: str = "latest",
iter: int = 0,
db_session: Session = Depends(deps.get_db_session),
):
data = get_db().read_artifact(db_session, key, tag=tag, iter=iter, project=project)
return {
"data": data,
}
def get_log(
db_session: Session,
project: str,
uid: str,
size: int = -1,
offset: int = 0,
source: LogSources = LogSources.AUTO,
):
out = b""
log_file = log_path(project, uid)
status = None
if log_file.exists() and source in [
LogSources.AUTO, LogSources.PERSISTENCY
]:
with log_file.open("rb") as fp:
fp.seek(offset)
out = fp.read(size)
status = ""
elif source in [LogSources.AUTO, LogSources.K8S]:
data = get_db().read_run(db_session, uid, project)
if not data:
log_and_raise(HTTPStatus.NOT_FOUND, project=project, uid=uid)
status = get_in(data, "status.state", "")
if get_k8s():
pods = get_k8s().get_logger_pods(uid)
if pods:
pod, new_status = list(pods.items())[0]
new_status = new_status.lower()
# TODO: handle in cron/tracking
if new_status != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
if status == "running":
now = now_date().isoformat()
update_in(data, "status.last_update", now)
if new_status == "failed":
update_in(data, "status.state", "error")
update_in(data, "status.error",
"error, check logs")
get_db().store_run(db_session, data, uid,
project)
if new_status == "succeeded":
update_in(data, "status.state", "completed")
get_db().store_run(db_session, data, uid,
project)
status = new_status
elif status == "running":
update_in(data, "status.state", "error")
update_in(data, "status.error",
"pod not found, maybe terminated")
get_db().store_run(db_session, data, uid, project)
status = "failed"
return out, status
def delete_runtime(kind: str,
label_selector: str = None,
force: bool = False,
db_session: Session = Depends(deps.get_db_session)):
if kind not in RuntimeKinds.runtime_with_handlers():
log_and_raise(status.HTTP_400_BAD_REQUEST,
kind=kind,
err='Invalid runtime kind')
runtime_handler = get_runtime_handler(kind)
runtime_handler.delete_resources(get_db(), db_session, label_selector,
force)
return Response(status_code=status.HTTP_204_NO_CONTENT)
async def update_run(
request: Request,
project: str,
uid: str,
iter: int = 0,
db_session: Session = Depends(deps.get_db_session)):
data = None
try:
data = await request.json()
except ValueError:
log_and_raise(HTTPStatus.BAD_REQUEST, reason="bad JSON body")
logger.debug(data)
await run_in_threadpool(get_db().update_run, db_session, data, uid, project, iter=iter)
logger.info("update run: {}".format(data))
return {}
def list_projects(full: str = "no",
db_session: Session = Depends(deps.get_db_session)):
full = strtobool(full)
fn = db2dict if full else attrgetter("name")
projects = []
for p in get_db().list_projects(db_session):
if isinstance(p, dict):
if full:
projects.append(p)
else:
projects.append(p.get("name"))
else:
projects.append(fn(p))
return {
"projects": projects,
}
def list_schedules(db_session: Session = Depends(deps.get_db_session)):
schedules = get_db().list_schedules(db_session)
return {
"schedules": list(schedules),
}
