async def file_diff(self, a: str, b: str) -> Apireturn: """ Diff the two files identified with the two hashes """ if a == "" or a == "0": a_lines: List[str] = [] else: a_path = os.path.join(self.server_slice._server_storage["files"], a) if not os.path.exists(a_path): raise NotFound() with open(a_path, "r", encoding="utf-8") as fd: a_lines = fd.readlines() if b == "" or b == "0": b_lines: List[str] = [] else: b_path = os.path.join(self.server_slice._server_storage["files"], b) if not os.path.exists(b_path): raise NotFound() with open(b_path, "r", encoding="utf-8") as fd: b_lines = fd.readlines() try: diff = difflib.unified_diff(a_lines, b_lines, fromfile=a, tofile=b) except FileNotFoundError: raise NotFound() return 200, {"diff": list(diff)}
async def dryrun_report(self, env: data.Environment, dryrun_id: uuid.UUID) -> Apireturn: dryrun = await data.DryRun.get_by_id(dryrun_id) if dryrun is None: raise NotFound("The given dryrun does not exist!") return 200, {"dryrun": dryrun}
async def get_compile_data( self, compile_id: uuid.UUID) -> Optional[model.CompileData]: compile: Optional[data.Compile] = await data.Compile.get_by_id( compile_id) if compile is None: raise NotFound("The given compile id does not exist") return compile.to_dto().compile_data
async def list_dryruns(self, env: data.Environment, version: int) -> List[DryRun]: model = await data.ConfigurationModel.get_version(environment=env.id, version=version) if model is None: raise NotFound("The requested version does not exist.") dtos = await data.DryRun.list_dryruns(order_by_column="date", order="DESC", environment=env.id, model=version) return dtos
async def dryrun_list(self, env: data.Environment, version: Optional[int] = None) -> Apireturn: query_args = {} query_args["environment"] = env.id if version is not None: model = await data.ConfigurationModel.get_version( environment=env.id, version=version) if model is None: raise NotFound("The request version does not exist.") query_args["model"] = version dryruns = await data.DryRun.get_list(**query_args) return ( 200, { "dryruns": [{ "id": x.id, "version": x.model, "date": x.date, "total": x.total, "todo": x.todo } for x in dryruns] }, )
async def get_fact(self, env: data.Environment, rid: ResourceIdStr, id: uuid.UUID) -> Fact: param = await data.Parameter.get_one(environment=env.id, resource_id=rid, id=id) if not param: raise NotFound(f"Fact with id {id} does not exist") return param.as_fact()
async def dryrun_trigger(self, env: data.Environment, version: int) -> uuid.UUID: model = await data.ConfigurationModel.get_version(environment=env.id, version=version) if model is None: raise NotFound("The requested version does not exist.") dryrun = await self.create_dryrun(env, version, model) return dryrun.id
async def environment_get(self, environment_id: uuid.UUID, details: bool = False) -> model.Environment: env = await data.Environment.get_by_id(environment_id, details=details) if env is None: raise NotFound("The environment id does not exist.") return env.to_dto()
async def get_notification( self, env: data.Environment, notification_id: uuid.UUID, ) -> Notification: notification = await data.Notification.get_one(environment=env.id, id=notification_id) if not notification: raise NotFound(f"Notification with id {notification_id} not found") return notification.to_dto()
async def environment_modify( self, environment_id: uuid.UUID, name: str, repository: Optional[str], branch: Optional[str], project_id: Optional[uuid.UUID] = None, description: Optional[str] = None, icon: Optional[str] = None, ) -> model.Environment: env = await data.Environment.get_by_id(environment_id) if env is None: raise NotFound("The environment id does not exist.") original_env = env.to_dto() project = project_id or env.project # check if an environment with this name is already defined in this project envs = await data.Environment.get_list(project=project, name=name) if len(envs) > 0 and envs[0].id != environment_id: raise BadRequest( f"Project with id={project} already has an environment with name {name}" ) fields = {"name": name} if repository is not None: fields["repo_url"] = repository if branch is not None: fields["repo_branch"] = branch # Update the project field if requested and the project exists if project_id is not None: project_from_db = await data.Project.get_by_id(project_id) if not project_from_db: raise BadRequest(f"Project with id={project_id} doesn't exist") fields["project"] = project_id if description is not None: fields["description"] = description if icon is not None: self.validate_icon(icon) fields["icon"] = icon try: await env.update_fields(connection=None, **fields) except StringDataRightTruncationError: raise BadRequest( "Maximum size of the icon data url or the description exceeded" ) await self.notify_listeners(EnvironmentAction.updated, env.to_dto(), original_env) return env.to_dto()
async def delete_setting(self, env: data.Environment, key: str) -> Apireturn: try: original_env = env.to_dto() await env.unset(key) warnings = await self._setting_change(env, key) await self.notify_listeners(EnvironmentAction.updated, env.to_dto(), original_env) return attach_warnings(200, None, warnings) except KeyError: raise NotFound()
async def set_setting(self, env: data.Environment, key: str, value: model.EnvSettingType) -> Apireturn: try: original_env = env.to_dto() await env.set(key, value) warnings = await self._setting_change(env, key) await self.notify_listeners(EnvironmentAction.updated, env.to_dto(), original_env) return attach_warnings(200, None, warnings) except KeyError: raise NotFound() except ValueError as e: raise ServerError(f"Invalid value. {e}")
async def project_get(self, project_id: uuid.UUID) -> model.Project: project = await data.Project.get_by_id(project_id) if project is None: raise NotFound("The project with given id does not exist.") project_model = project.to_dto() project_model.environments = [ e.to_dto() for e in await data.Environment.get_list(project=project_id) ] return project_model
async def project_modify(self, project_id: uuid.UUID, name: str) -> model.Project: try: project = await data.Project.get_by_id(project_id) if project is None: raise NotFound("The project with given id does not exist.") await project.update_fields(name=name) return project.to_dto() except asyncpg.exceptions.UniqueViolationError: raise ServerError(f"A project with name {name} already exists.")
async def environment_setting_get( self, env: data.Environment, key: str) -> model.EnvironmentSettingsReponse: try: value = await env.get(key) return model.EnvironmentSettingsReponse( settings={key: value}, definition={ k: v.to_dto() for k, v in data.Environment._settings.items() }) except KeyError: raise NotFound()
async def project_delete(self, project_id: uuid.UUID) -> None: project = await data.Project.get_by_id(project_id) if project is None: raise NotFound("The project with given id does not exist.") environments = await data.Environment.get_list(project=project.id) for env in environments: await asyncio.gather( self.autostarted_agent_manager.stop_agents(env), env.delete_cascade()) self.resource_service.close_resource_action_logger(env.id) await project.delete()
async def environment_create( self, project_id: uuid.UUID, name: str, repository: str, branch: str, environment_id: Optional[uuid.UUID], description: str = "", icon: str = "", ) -> model.Environment: if environment_id is None: environment_id = uuid.uuid4() if (repository is None and branch is not None) or (repository is not None and branch is None): raise BadRequest("Repository and branch should be set together.") # fetch the project first project = await data.Project.get_by_id(project_id) if project is None: raise NotFound( "The project id for the environment does not exist.") # check if an environment with this name is already defined in this project envs = await data.Environment.get_list(project=project_id, name=name) if len(envs) > 0: raise ServerError( f"Project {project.name} (id={project.id}) already has an environment with name {name}" ) self.validate_icon(icon) env = data.Environment( id=environment_id, name=name, project=project_id, repo_url=repository, repo_branch=branch, description=description, icon=icon, ) try: await env.insert() except StringDataRightTruncationError: raise BadRequest( "Maximum size of the icon data url or the description exceeded" ) await self.notify_listeners(EnvironmentAction.created, env.to_dto()) return env.to_dto()
async def environment_setting_delete(self, env: data.Environment, key: str) -> ReturnValue[None]: try: original_env = env.to_dto() await env.unset(key) warnings = await self._setting_change(env, key) result: ReturnValue[None] = ReturnValue(response=None) if warnings: result.add_warnings(warnings) await self.notify_listeners(EnvironmentAction.updated, env.to_dto(), original_env) return result except KeyError: raise NotFound()
async def environment_delete(self, environment_id: uuid.UUID) -> None: env = await data.Environment.get_by_id(environment_id) if env is None: raise NotFound("The environment with given id does not exist.") is_protected_environment = await env.get(data.PROTECTED_ENVIRONMENT) if is_protected_environment: raise Forbidden( f"Environment {environment_id} is protected. See environment setting: {data.PROTECTED_ENVIRONMENT}" ) await asyncio.gather(self.autostarted_agent_manager.stop_agents(env), env.delete_cascade()) self.resource_service.close_resource_action_logger(environment_id) await self.notify_listeners(EnvironmentAction.deleted, env.to_dto())
async def environment_settings_set( self, env: data.Environment, key: str, value: model.EnvSettingType) -> ReturnValue[None]: try: original_env = env.to_dto() await env.set(key, value) warnings = await self._setting_change(env, key) result: ReturnValue[None] = ReturnValue(response=None) if warnings: result.add_warnings(warnings) await self.notify_listeners(EnvironmentAction.updated, env.to_dto(), original_env) return result except KeyError: raise NotFound() except ValueError: raise ServerError("Invalid value")
async def update_notification( self, env: data.Environment, notification_id: uuid.UUID, read: Optional[bool] = None, cleared: Optional[bool] = None, ) -> Notification: notification = await data.Notification.get_one(environment=env.id, id=notification_id) if not notification: raise NotFound(f"Notification with id {notification_id} not found") if read is not None and cleared is not None: await notification.update(read=read, cleared=cleared) elif read is not None: await notification.update(read=read) elif cleared is not None: await notification.update(cleared=cleared) else: raise BadRequest("At least one of {read, cleared} should be specified for a valid update") return notification.to_dto()
async def halt(self, env: data.Environment) -> None: async with self.agent_state_lock: async with data.Environment.get_connection() as connection: async with connection.transaction(): refreshed_env: Optional[ data.Environment] = await data.Environment.get_by_id( env.id, connection=connection) if refreshed_env is None: raise NotFound("Environment %s does not exist" % env.id) # silently ignore requests if this environment has already been halted if refreshed_env.halted: return await refreshed_env.update_fields(halted=True, connection=connection) await self.agent_manager.halt_agents(refreshed_env, connection=connection) await self.autostarted_agent_manager.stop_agents(refreshed_env)
async def get_code(self, env: data.Environment, code_id: int, resource: str) -> Apireturn: code = await data.Code.get_version(environment=env.id, version=code_id, resource=resource) if code is None: raise NotFound("The version of the code does not exist.") sources = {} if code.source_refs is not None: for code_hash, (file_name, module, req) in code.source_refs.items(): content = self.file_slice.get_file_internal(code_hash) sources[code_hash] = (file_name, module, content.decode(), req) return 200, { "version": code_id, "environment": env.id, "resource": resource, "sources": sources }
async def dryrun_diff(self, env: data.Environment, version: int, report_id: uuid.UUID) -> DryRunReport: dryrun = await data.DryRun.get_one(environment=env.id, model=version, id=report_id) if dryrun is None: raise NotFound("The given dryrun does not exist!") resources = dryrun.to_dict()["resources"] from_resources = {} to_resources = {} resources_with_already_known_status = { resource_version_id: resource for resource_version_id, resource in resources.items() if resource.get("diff_status") } resources_to_diff = { resource_version_id: resource for resource_version_id, resource in resources.items() if resource_version_id not in resources_with_already_known_status.keys() } for resource_version_id, resource in resources_to_diff.items(): resource_id = Id.parse_id(resource_version_id).resource_str() from_attributes = self.get_attributes_from_changes(resource["changes"], "current") to_attributes = self.get_attributes_from_changes(resource["changes"], "desired") from_resources[resource_id] = diff.Resource(resource_id, from_attributes) to_resources[resource_id] = diff.Resource(resource_id, to_attributes) if "purged" in resource["changes"]: if self.resource_will_be_unpurged(from_attributes, to_attributes): from_resources.pop(resource_id) if self.resource_will_be_purged(from_attributes, to_attributes): to_resources.pop(resource_id) version_diff = diff.generate_diff(from_resources, to_resources, include_unmodified=True) version_diff += [ ResourceDiff( resource_id=Id.parse_resource_version_id(rvid).resource_str(), attributes={}, status=resource.get("diff_status") ) for rvid, resource in resources_with_already_known_status.items() ] version_diff.sort(key=lambda r: r.resource_id) dto = DryRunReport(summary=dryrun.to_dto(), diff=version_diff) return dto
def get_file_internal(self, file_hash: str) -> bytes: """get_file, but on return code 200, content is not encoded """ file_name = os.path.join(self.server_slice._server_storage["files"], file_hash) if not os.path.exists(file_name): raise NotFound() with open(file_name, "rb") as fd: content = fd.read() actualhash = hash_file(content) if actualhash == file_hash: return content # handle corrupt file if opt.server_delete_currupt_files.get(): LOGGER.error( "File corrupt, expected hash %s but found %s at %s, Deleting file", file_hash, actualhash, file_name) try: os.remove(file_name) except OSError: LOGGER.exception("Failed to delete file %s", file_name) raise ServerError( f"File corrupt, expected hash {file_hash} but found {actualhash}. Failed to delete file, please " "contact the server administrator") raise ServerError( f"File corrupt, expected hash {file_hash} but found {actualhash}. " "Deleting file, please re-upload the corrupt file.") else: LOGGER.error( "File corrupt, expected hash %s but found %s at %s", file_hash, actualhash, file_name) raise ServerError( f"File corrupt, expected hash {file_hash} but found {actualhash}, please contact the server administrator" )
async def get_environment(self, environment_id: uuid.UUID, versions: Optional[int] = None, resources: Optional[int] = None) -> Apireturn: versions = 0 if versions is None else int(versions) resources = 0 if resources is None else int(resources) env = await data.Environment.get_by_id(environment_id) if env is None: raise NotFound("The environment id does not exist.") env_dict = env.to_dict() if versions > 0: env_dict["versions"] = await data.ConfigurationModel.get_versions( environment_id, limit=versions) if resources > 0: env_dict["resources"] = await data.Resource.get_resources_report( environment=environment_id) return 200, {"environment": env_dict}
async def compile_details(self, env: data.Environment, id: uuid.UUID) -> model.CompileDetails: details = await data.Compile.get_compile_details(env.id, id) if not details: raise NotFound("The compile with the given id does not exist.") return details
async def _check_version_exists(self, env: uuid.UUID, version: int) -> None: version_object = await data.ConfigurationModel.get_version( env, version) if not version_object: raise NotFound(f"Version {version} not found")