async def file_diff(self, a: str, b: str) -> Apireturn:
"""
Diff the two files identified with the two hashes
"""
if a == "" or a == "0":
a_lines: List[str] = []
else:
a_path = os.path.join(self.server_slice._server_storage["files"],
a)
if not os.path.exists(a_path):
raise NotFound()
with open(a_path, "r", encoding="utf-8") as fd:
a_lines = fd.readlines()
if b == "" or b == "0":
b_lines: List[str] = []
else:
b_path = os.path.join(self.server_slice._server_storage["files"],
b)
if not os.path.exists(b_path):
raise NotFound()
with open(b_path, "r", encoding="utf-8") as fd:
b_lines = fd.readlines()
try:
diff = difflib.unified_diff(a_lines, b_lines, fromfile=a, tofile=b)
except FileNotFoundError:
raise NotFound()
return 200, {"diff": list(diff)}
async def dryrun_report(self, env: data.Environment,
dryrun_id: uuid.UUID) -> Apireturn:
dryrun = await data.DryRun.get_by_id(dryrun_id)
if dryrun is None:
raise NotFound("The given dryrun does not exist!")
return 200, {"dryrun": dryrun}
async def get_compile_data(
self, compile_id: uuid.UUID) -> Optional[model.CompileData]:
compile: Optional[data.Compile] = await data.Compile.get_by_id(
compile_id)
if compile is None:
raise NotFound("The given compile id does not exist")
return compile.to_dto().compile_data
async def list_dryruns(self, env: data.Environment, version: int) -> List[DryRun]:
model = await data.ConfigurationModel.get_version(environment=env.id, version=version)
if model is None:
raise NotFound("The requested version does not exist.")
dtos = await data.DryRun.list_dryruns(order_by_column="date", order="DESC", environment=env.id, model=version)
return dtos
async def dryrun_list(self,
env: data.Environment,
version: Optional[int] = None) -> Apireturn:
query_args = {}
query_args["environment"] = env.id
if version is not None:
model = await data.ConfigurationModel.get_version(
environment=env.id, version=version)
if model is None:
raise NotFound("The request version does not exist.")
query_args["model"] = version
dryruns = await data.DryRun.get_list(**query_args)
return (
200,
{
"dryruns": [{
"id": x.id,
"version": x.model,
"date": x.date,
"total": x.total,
"todo": x.todo
} for x in dryruns]
},
)
async def get_fact(self, env: data.Environment, rid: ResourceIdStr,
id: uuid.UUID) -> Fact:
param = await data.Parameter.get_one(environment=env.id,
resource_id=rid,
id=id)
if not param:
raise NotFound(f"Fact with id {id} does not exist")
return param.as_fact()
async def dryrun_trigger(self, env: data.Environment, version: int) -> uuid.UUID:
model = await data.ConfigurationModel.get_version(environment=env.id, version=version)
if model is None:
raise NotFound("The requested version does not exist.")
dryrun = await self.create_dryrun(env, version, model)
return dryrun.id
async def environment_get(self,
environment_id: uuid.UUID,
details: bool = False) -> model.Environment:
env = await data.Environment.get_by_id(environment_id, details=details)
if env is None:
raise NotFound("The environment id does not exist.")
return env.to_dto()
async def get_notification(
self,
env: data.Environment,
notification_id: uuid.UUID,
) -> Notification:
notification = await data.Notification.get_one(environment=env.id, id=notification_id)
if not notification:
raise NotFound(f"Notification with id {notification_id} not found")
return notification.to_dto()
async def environment_modify(
self,
environment_id: uuid.UUID,
name: str,
repository: Optional[str],
branch: Optional[str],
project_id: Optional[uuid.UUID] = None,
description: Optional[str] = None,
icon: Optional[str] = None,
) -> model.Environment:
env = await data.Environment.get_by_id(environment_id)
if env is None:
raise NotFound("The environment id does not exist.")
original_env = env.to_dto()
project = project_id or env.project
# check if an environment with this name is already defined in this project
envs = await data.Environment.get_list(project=project, name=name)
if len(envs) > 0 and envs[0].id != environment_id:
raise BadRequest(
f"Project with id={project} already has an environment with name {name}"
)
fields = {"name": name}
if repository is not None:
fields["repo_url"] = repository
if branch is not None:
fields["repo_branch"] = branch
# Update the project field if requested and the project exists
if project_id is not None:
project_from_db = await data.Project.get_by_id(project_id)
if not project_from_db:
raise BadRequest(f"Project with id={project_id} doesn't exist")
fields["project"] = project_id
if description is not None:
fields["description"] = description
if icon is not None:
self.validate_icon(icon)
fields["icon"] = icon
try:
await env.update_fields(connection=None, **fields)
except StringDataRightTruncationError:
raise BadRequest(
"Maximum size of the icon data url or the description exceeded"
)
await self.notify_listeners(EnvironmentAction.updated, env.to_dto(),
original_env)
return env.to_dto()
async def delete_setting(self, env: data.Environment,
key: str) -> Apireturn:
try:
original_env = env.to_dto()
await env.unset(key)
warnings = await self._setting_change(env, key)
await self.notify_listeners(EnvironmentAction.updated,
env.to_dto(), original_env)
return attach_warnings(200, None, warnings)
except KeyError:
raise NotFound()
async def set_setting(self, env: data.Environment, key: str,
value: model.EnvSettingType) -> Apireturn:
try:
original_env = env.to_dto()
await env.set(key, value)
warnings = await self._setting_change(env, key)
await self.notify_listeners(EnvironmentAction.updated,
env.to_dto(), original_env)
return attach_warnings(200, None, warnings)
except KeyError:
raise NotFound()
except ValueError as e:
raise ServerError(f"Invalid value. {e}")
async def project_get(self, project_id: uuid.UUID) -> model.Project:
project = await data.Project.get_by_id(project_id)
if project is None:
raise NotFound("The project with given id does not exist.")
project_model = project.to_dto()
project_model.environments = [
e.to_dto()
for e in await data.Environment.get_list(project=project_id)
]
return project_model
async def project_modify(self, project_id: uuid.UUID,
name: str) -> model.Project:
try:
project = await data.Project.get_by_id(project_id)
if project is None:
raise NotFound("The project with given id does not exist.")
await project.update_fields(name=name)
return project.to_dto()
except asyncpg.exceptions.UniqueViolationError:
raise ServerError(f"A project with name {name} already exists.")
async def environment_setting_get(
self, env: data.Environment,
key: str) -> model.EnvironmentSettingsReponse:
try:
value = await env.get(key)
return model.EnvironmentSettingsReponse(
settings={key: value},
definition={
k: v.to_dto()
for k, v in data.Environment._settings.items()
})
except KeyError:
raise NotFound()
async def project_delete(self, project_id: uuid.UUID) -> None:
project = await data.Project.get_by_id(project_id)
if project is None:
raise NotFound("The project with given id does not exist.")
environments = await data.Environment.get_list(project=project.id)
for env in environments:
await asyncio.gather(
self.autostarted_agent_manager.stop_agents(env),
env.delete_cascade())
self.resource_service.close_resource_action_logger(env.id)
await project.delete()
async def environment_create(
self,
project_id: uuid.UUID,
name: str,
repository: str,
branch: str,
environment_id: Optional[uuid.UUID],
description: str = "",
icon: str = "",
) -> model.Environment:
if environment_id is None:
environment_id = uuid.uuid4()
if (repository is None
and branch is not None) or (repository is not None
and branch is None):
raise BadRequest("Repository and branch should be set together.")
# fetch the project first
project = await data.Project.get_by_id(project_id)
if project is None:
raise NotFound(
"The project id for the environment does not exist.")
# check if an environment with this name is already defined in this project
envs = await data.Environment.get_list(project=project_id, name=name)
if len(envs) > 0:
raise ServerError(
f"Project {project.name} (id={project.id}) already has an environment with name {name}"
)
self.validate_icon(icon)
env = data.Environment(
id=environment_id,
name=name,
project=project_id,
repo_url=repository,
repo_branch=branch,
description=description,
icon=icon,
)
try:
await env.insert()
except StringDataRightTruncationError:
raise BadRequest(
"Maximum size of the icon data url or the description exceeded"
)
await self.notify_listeners(EnvironmentAction.created, env.to_dto())
return env.to_dto()
async def environment_setting_delete(self, env: data.Environment,
key: str) -> ReturnValue[None]:
try:
original_env = env.to_dto()
await env.unset(key)
warnings = await self._setting_change(env, key)
result: ReturnValue[None] = ReturnValue(response=None)
if warnings:
result.add_warnings(warnings)
await self.notify_listeners(EnvironmentAction.updated,
env.to_dto(), original_env)
return result
except KeyError:
raise NotFound()
async def environment_delete(self, environment_id: uuid.UUID) -> None:
env = await data.Environment.get_by_id(environment_id)
if env is None:
raise NotFound("The environment with given id does not exist.")
is_protected_environment = await env.get(data.PROTECTED_ENVIRONMENT)
if is_protected_environment:
raise Forbidden(
f"Environment {environment_id} is protected. See environment setting: {data.PROTECTED_ENVIRONMENT}"
)
await asyncio.gather(self.autostarted_agent_manager.stop_agents(env),
env.delete_cascade())
self.resource_service.close_resource_action_logger(environment_id)
await self.notify_listeners(EnvironmentAction.deleted, env.to_dto())
async def environment_settings_set(
self, env: data.Environment, key: str,
value: model.EnvSettingType) -> ReturnValue[None]:
try:
original_env = env.to_dto()
await env.set(key, value)
warnings = await self._setting_change(env, key)
result: ReturnValue[None] = ReturnValue(response=None)
if warnings:
result.add_warnings(warnings)
await self.notify_listeners(EnvironmentAction.updated,
env.to_dto(), original_env)
return result
except KeyError:
raise NotFound()
except ValueError:
raise ServerError("Invalid value")
async def update_notification(
self,
env: data.Environment,
notification_id: uuid.UUID,
read: Optional[bool] = None,
cleared: Optional[bool] = None,
) -> Notification:
notification = await data.Notification.get_one(environment=env.id, id=notification_id)
if not notification:
raise NotFound(f"Notification with id {notification_id} not found")
if read is not None and cleared is not None:
await notification.update(read=read, cleared=cleared)
elif read is not None:
await notification.update(read=read)
elif cleared is not None:
await notification.update(cleared=cleared)
else:
raise BadRequest("At least one of {read, cleared} should be specified for a valid update")
return notification.to_dto()
async def halt(self, env: data.Environment) -> None:
async with self.agent_state_lock:
async with data.Environment.get_connection() as connection:
async with connection.transaction():
refreshed_env: Optional[
data.Environment] = await data.Environment.get_by_id(
env.id, connection=connection)
if refreshed_env is None:
raise NotFound("Environment %s does not exist" %
env.id)
# silently ignore requests if this environment has already been halted
if refreshed_env.halted:
return
await refreshed_env.update_fields(halted=True,
connection=connection)
await self.agent_manager.halt_agents(refreshed_env,
connection=connection)
await self.autostarted_agent_manager.stop_agents(refreshed_env)
async def get_code(self, env: data.Environment, code_id: int,
resource: str) -> Apireturn:
code = await data.Code.get_version(environment=env.id,
version=code_id,
resource=resource)
if code is None:
raise NotFound("The version of the code does not exist.")
sources = {}
if code.source_refs is not None:
for code_hash, (file_name, module,
req) in code.source_refs.items():
content = self.file_slice.get_file_internal(code_hash)
sources[code_hash] = (file_name, module, content.decode(), req)
return 200, {
"version": code_id,
"environment": env.id,
"resource": resource,
"sources": sources
}
async def dryrun_diff(self, env: data.Environment, version: int, report_id: uuid.UUID) -> DryRunReport:
dryrun = await data.DryRun.get_one(environment=env.id, model=version, id=report_id)
if dryrun is None:
raise NotFound("The given dryrun does not exist!")
resources = dryrun.to_dict()["resources"]
from_resources = {}
to_resources = {}
resources_with_already_known_status = {
resource_version_id: resource for resource_version_id, resource in resources.items() if resource.get("diff_status")
}
resources_to_diff = {
resource_version_id: resource
for resource_version_id, resource in resources.items()
if resource_version_id not in resources_with_already_known_status.keys()
}
for resource_version_id, resource in resources_to_diff.items():
resource_id = Id.parse_id(resource_version_id).resource_str()
from_attributes = self.get_attributes_from_changes(resource["changes"], "current")
to_attributes = self.get_attributes_from_changes(resource["changes"], "desired")
from_resources[resource_id] = diff.Resource(resource_id, from_attributes)
to_resources[resource_id] = diff.Resource(resource_id, to_attributes)
if "purged" in resource["changes"]:
if self.resource_will_be_unpurged(from_attributes, to_attributes):
from_resources.pop(resource_id)
if self.resource_will_be_purged(from_attributes, to_attributes):
to_resources.pop(resource_id)
version_diff = diff.generate_diff(from_resources, to_resources, include_unmodified=True)
version_diff += [
ResourceDiff(
resource_id=Id.parse_resource_version_id(rvid).resource_str(), attributes={}, status=resource.get("diff_status")
)
for rvid, resource in resources_with_already_known_status.items()
]
version_diff.sort(key=lambda r: r.resource_id)
dto = DryRunReport(summary=dryrun.to_dto(), diff=version_diff)
return dto
def get_file_internal(self, file_hash: str) -> bytes:
"""get_file, but on return code 200, content is not encoded """
file_name = os.path.join(self.server_slice._server_storage["files"],
file_hash)
if not os.path.exists(file_name):
raise NotFound()
with open(file_name, "rb") as fd:
content = fd.read()
actualhash = hash_file(content)
if actualhash == file_hash:
return content
# handle corrupt file
if opt.server_delete_currupt_files.get():
LOGGER.error(
"File corrupt, expected hash %s but found %s at %s, Deleting file",
file_hash, actualhash, file_name)
try:
os.remove(file_name)
except OSError:
LOGGER.exception("Failed to delete file %s", file_name)
raise ServerError(
f"File corrupt, expected hash {file_hash} but found {actualhash}. Failed to delete file, please "
"contact the server administrator")
raise ServerError(
f"File corrupt, expected hash {file_hash} but found {actualhash}. "
"Deleting file, please re-upload the corrupt file.")
else:
LOGGER.error(
"File corrupt, expected hash %s but found %s at %s",
file_hash, actualhash, file_name)
raise ServerError(
f"File corrupt, expected hash {file_hash} but found {actualhash}, please contact the server administrator"
)
async def get_environment(self,
environment_id: uuid.UUID,
versions: Optional[int] = None,
resources: Optional[int] = None) -> Apireturn:
versions = 0 if versions is None else int(versions)
resources = 0 if resources is None else int(resources)
env = await data.Environment.get_by_id(environment_id)
if env is None:
raise NotFound("The environment id does not exist.")
env_dict = env.to_dict()
if versions > 0:
env_dict["versions"] = await data.ConfigurationModel.get_versions(
environment_id, limit=versions)
if resources > 0:
env_dict["resources"] = await data.Resource.get_resources_report(
environment=environment_id)
return 200, {"environment": env_dict}
async def compile_details(self, env: data.Environment,
id: uuid.UUID) -> model.CompileDetails:
details = await data.Compile.get_compile_details(env.id, id)
if not details:
raise NotFound("The compile with the given id does not exist.")
return details
async def _check_version_exists(self, env: uuid.UUID,
version: int) -> None:
version_object = await data.ConfigurationModel.get_version(
env, version)
if not version_object:
raise NotFound(f"Version {version} not found")
