def reset_problem_config( lakefs_reset: schemas.LakeFSReset, problem: models.Problem = Depends(parse_problem), ) -> StandardResponse[Empty]: problem_config = LakeFSProblemConfig(problem) problem_config.reset(lakefs_reset) return StandardResponse()
def get_file_or_directory_info_in_uncommitted_problem_config( path: str = Path(...), problem: models.Problem = Depends(parse_problem), ) -> StandardResponse[FileInfo]: problem_config = LakeFSProblemConfig(problem) file_info = problem_config.get_file_info(pathlib.Path(path)) return StandardResponse(file_info)
def delete_file_from_uncommitted_problem_config( path: str = Path(...), problem: models.Problem = Depends(parse_problem), ) -> StandardResponse[FileInfo]: problem_config = LakeFSProblemConfig(problem) file_info = problem_config.delete_file(pathlib.Path(path)) return StandardResponse(file_info)
def upload_file_to_problem_config( file: UploadFile = File(...), problem: models.Problem = Depends(parse_problem), path: str = Depends(parse_file_path), ) -> StandardResponse[FileInfo]: problem_config = LakeFSProblemConfig(problem) file_info = problem_config.upload_file(pathlib.Path(path), file.file) return StandardResponse(file_info)
def update_problem_config_by_archive( file: UploadFile = File(...), problem: models.Problem = Depends(parse_problem) ) -> StandardResponse[Empty]: logger.info("problem config archive name: %s", file.filename) problem_config = LakeFSProblemConfig(problem) problem_config.upload_archive(file.filename, file.file) return StandardResponse()
def delete_directory_from_uncommitted_problem_config( path: str = Path(...), problem: models.Problem = Depends(parse_problem), recursive: bool = Query( False, description="Act as -r in the rm command. " "If false, only empty directory can be deleted.", ), ) -> StandardResponse[FileInfo]: problem_config = LakeFSProblemConfig(problem) file_info = problem_config.delete_directory(pathlib.Path(path), recursive) return StandardResponse(file_info)
async def create_problem( problem_create: schemas.ProblemCreate, background_tasks: BackgroundTasks, domain: models.Domain = Depends(parse_domain_from_auth), user: models.User = Depends(parse_user_from_auth), session: AsyncSession = Depends(db_session_dependency), ) -> StandardResponse[schemas.ProblemDetail]: try: problem_group = models.ProblemGroup() session.sync_session.add(problem_group) logger.info(f"problem group created: {problem_group}") problem = models.Problem( **problem_create.dict(), domain_id=domain.id, owner_id=user.id, problem_group_id=problem_group.id, ) session.sync_session.add(problem) logger.info(f"problem created: {problem}") await session.commit() await session.refresh(problem) except Exception as e: logger.exception(f"problem creation failed: {problem_create}") raise e lakefs_problem_config = LakeFSProblemConfig(problem) background_tasks.add_task(lakefs_problem_config.ensure_branch) return StandardResponse(problem)
def download_file_in_problem_config( path: str = Path(...), problem: models.Problem = Depends(parse_problem), config: Optional[models.ProblemConfig] = Depends(parse_problem_config), ) -> Any: problem_config = LakeFSProblemConfig(problem) ref: Optional[str] if config is not None: ref = config.commit_id else: ref = None file = problem_config.download_file(pathlib.Path(path), ref) response = StreamingResponse(file) filename = pathlib.Path(path).name response.content_disposition = f'attachment; filename="{filename}"' return response
def download_problem_config_archive( temp_dir: pathlib.Path = Depends(TemporaryDirectory()), archive_format: ArchiveType = Query(ArchiveType.zip), problem: models.Problem = Depends(parse_problem), config: Optional[models.ProblemConfig] = Depends(parse_problem_config), ) -> Any: # use lakefs to sync and zip files ref: Optional[str] if config is not None: ref = config.commit_id else: ref = None problem_config = LakeFSProblemConfig(problem) file_path = problem_config.download_archive(temp_dir, archive_format, ref) # TODO: cache the archive response = StreamingResponse(iter_file(file_path)) response.content_disposition = f'attachment; filename="{file_path.name}"' return response
async def claim_record_by_judge( judge_claim: schemas.JudgeClaim, record: models.Record = Depends(parse_record_judger), user: models.User = Depends(parse_user_from_auth), ) -> StandardResponse[schemas.JudgeCredentials]: # task_id can only be obtained by listening to the celery task queue # we give the worker with task_id the chance to claim the record # celery tasks can be retried, only one worker can hold the task_id at the same time # if a rejudge is scheduled, task_id changes, so previous task will be ineffective # TODO: we probably need a lock to handle race condition of rejudge and claim if record.task_id is None or record.task_id != judge_claim.task_id: raise BizError(ErrorCode.Error) # if record.state not in (schemas.RecordState.queueing, schemas.RecordState.retrying): # raise BizError(ErrorCode.Error) # we can mark task failed if no problem config is available if record.problem_config is None or record.problem is None: raise BizError(ErrorCode.Error) # we always reset the state to "fetched", for both first attempt and retries record.judger_id = user.id record.state = schemas.RecordState.fetched await record.save_model() logger.info("judger claim record: {}", record) # initialize the permission of the judger to lakefs # the user have read access to all problems in the problem group, # actually only the access to one branch is necessary, # but it will create too many policies, so we grant all for simplicity # the user have read/write access to all records in the problem, # because the judger will write test result to the repo await record.fetch_related("problem") access_key = await models.UserAccessKey.get_lakefs_access_key(user) lakefs_problem_config = LakeFSProblemConfig(record.problem) lakefs_record = LakeFSRecord(record.problem, record) def sync_func() -> None: lakefs_problem_config.ensure_user_policy(user, "read") lakefs_record.ensure_user_policy(user, "all") await run_in_threadpool(sync_func) judge_credentials = schemas.JudgeCredentials( access_key_id=access_key.access_key_id, secret_access_key=access_key.secret_access_key, problem_config_repo_name=lakefs_problem_config.repo_name, problem_config_commit_id=record.problem_config.commit_id, record_repo_name=lakefs_record.repo_name, record_commit_id=record.commit_id, ) return StandardResponse(judge_credentials)
def sync_func() -> Commit: lakefs_problem_config = LakeFSProblemConfig(problem) # manager = Manager(logger, lakefs_problem_config.storage) # manager.validate_source() return lakefs_problem_config.commit(commit.message)