def get_file(self, hash_id: str) -> Optional[bytes]:
"""
Retrieve a file from the fileserver identified with the given id. The convention is to use the sha1sum of the
content to identify it.
:param hash_id: The id of the content/file to retrieve from the server.
:return: The content in the form of a bytestring or none is the content does not exist.
"""
def call() -> typing.Awaitable[Result]:
return self.get_client().get_file(hash_id)
result = self.run_sync(call)
if result.code == 404:
return None
elif result.result and result.code == 200:
file_contents = base64.b64decode(result.result["content"])
actual_hash_of_file = hash_file(file_contents)
if hash_id != actual_hash_of_file:
raise Exception(
f"File hash verification failed, expected: {hash_id} but got {actual_hash_of_file}"
)
return file_contents
else:
raise Exception("An error occurred while retrieving file %s" %
hash_id)
async def upload_code(self, env: data.Environment, code_id: int,
resource: str, sources: JsonType) -> Apireturn:
code = await data.Code.get_version(environment=env.id,
version=code_id,
resource=resource)
if code is not None:
raise ServerError(
"Code for this version has already been uploaded.")
hasherrors = any((k != hash_file(content[2].encode())
for k, content in sources.items()))
if hasherrors:
return 400, {
"message": "Hashes in source map do not match to source_code"
}
for file_hash in self.file_slice.stat_file_internal(sources.keys()):
self.file_slice.upload_file_internal(
file_hash, sources[file_hash][2].encode())
compact = {
code_hash: (file_name, module, req)
for code_hash, (file_name, module, _, req) in sources.items()
}
code = data.Code(environment=env.id,
version=code_id,
resource=resource,
source_refs=compact)
await code.insert()
return 200
def upload_file_internal(self, file_hash: str, content: bytes) -> None:
file_name = os.path.join(self.server_slice._server_storage["files"],
file_hash)
if os.path.exists(file_name):
raise ServerError("A file with this id already exists.")
if hash_file(content) != file_hash:
raise BadRequest("The hash does not match the content")
with open(file_name, "wb+") as fd:
fd.write(content)
def upload_file(self, content: Union[str, bytes]) -> str:
"""
Upload a file to the configuration server. This operation is not
executed in the transaction.
"""
bcontent: bytes
if not isinstance(content, bytes):
bcontent = content.encode("utf-8")
else:
bcontent = content
hash_id = hash_file(bcontent)
self._file_store[hash_id] = bcontent
return hash_id
def get_file_internal(self, file_hash: str) -> bytes:
"""get_file, but on return code 200, content is not encoded """
file_name = os.path.join(self.server_slice._server_storage["files"],
file_hash)
if not os.path.exists(file_name):
raise NotFound()
with open(file_name, "rb") as fd:
content = fd.read()
actualhash = hash_file(content)
if actualhash == file_hash:
return content
# handle corrupt file
if opt.server_delete_currupt_files.get():
LOGGER.error(
"File corrupt, expected hash %s but found %s at %s, Deleting file",
file_hash, actualhash, file_name)
try:
os.remove(file_name)
except OSError:
LOGGER.exception("Failed to delete file %s", file_name)
raise ServerError(
f"File corrupt, expected hash {file_hash} but found {actualhash}. Failed to delete file, please "
"contact the server administrator")
raise ServerError(
f"File corrupt, expected hash {file_hash} but found {actualhash}. "
"Deleting file, please re-upload the corrupt file.")
else:
LOGGER.error(
"File corrupt, expected hash %s but found %s at %s",
file_hash, actualhash, file_name)
raise ServerError(
f"File corrupt, expected hash {file_hash} but found {actualhash}, please contact the server administrator"
)
def make_source(collector, filename, module, source, req):
myhash = hash_file(source.encode())
collector[myhash] = [filename, module, source, req]
return collector