def sort_transcode_outputs_into_filegroup(
filegroup: pb2.FileGroup,
outputs: transcode.OutputsDict) -> List[Tuple[str, Path]]:
if len(outputs) == 0:
return []
best_output = sorted(set(outputs.keys()) - set("original"))[-1]
print(best_output)
if "original" not in outputs:
# replace the "main" data with the best output
filegroup.main.sdid = file_to_sdid(outputs[best_output])
filegroup.main.mime = document_files.guess_mimetype(
outputs[best_output])
# fill in media formats
files_to_move = []
filegroup.media_formats.clear()
for key, value in outputs.items():
if isinstance(key, TargetRepresentation):
str_key = key.to_str()
else:
str_key = key
data_id = file_to_sdid(value)
filegroup.media_formats[str_key].sdid = data_id
filegroup.media_formats[str_key].mime = document_files.guess_mimetype(
value)
files_to_move.append((data_id, value))
return files_to_move
def test_download_to(self) -> None:
lfs = LocalFileStorage(self.path / "store")
sdid = file_to_sdid(self.path / "test.txt")
lfs.upload(sdid, self.path / "test.txt")
lfs.download_to(sdid, self.path / "out.txt")
with (self.path / "out.txt").open("r") as f:
self.assertEqual("Hello, world!", f.read())
def test_delete(self) -> None:
sdid = file_to_sdid(self.path / "test.txt")
lfs = LocalFileStorage(self.path / "store")
lfs.upload(sdid, self.path / "test.txt")
self.assertTrue(lfs.contains(sdid))
lfs.delete(sdid)
self.assertFalse(lfs.contains(sdid))
def test_list(self) -> None:
lfs = LocalFileStorage(self.path / "store")
contents = list(lfs.list())
self.assertEqual([], contents)
sdid = file_to_sdid(self.path / "test.txt")
lfs.upload(sdid, self.path / "test.txt")
contents = list(lfs.list())
self.assertEqual([sdid], contents)
def test_upload_contains_stat(self) -> None:
lfs = LocalFileStorage(self.path / "store")
sdid = file_to_sdid(self.path / "test.txt")
lfs.upload(sdid, self.path / "test.txt")
self.assertTrue(lfs.contains(sdid))
stat = lfs.download_stat(sdid)
self.assertEqual(13, stat.size_bytes)
ts_diff = datetime.datetime.now() - stat.upload_time
self.assertTrue(ts_diff <= datetime.timedelta(minutes=1))
def test_download_bytes(self) -> None:
lfs = LocalFileStorage(self.path / "store")
sdid = file_to_sdid(self.path / "test.txt")
lfs.upload(sdid, self.path / "test.txt")
with lfs.download_bytes(sdid) as f:
self.assertEqual(b'Hello, world!', f.read())
with lfs.download_bytes(sdid, (0, 4)) as f:
self.assertEqual(b'Hello', f.read())
with lfs.download_bytes(sdid, (7, 12)) as f:
self.assertEqual(b'world!', f.read())
def test_basic(self) -> None:
sw = ScannerWorker(self.tempdir.name, self.state,
self.storage) # type: ignore
sw.scan_inbox()
self.assertEqual([], sw.to_ingest)
filepath = Path(self.tempdir.name) / "temp.txt"
with filepath.open("w") as f:
f.write("hello")
sdid = file_to_sdid(filepath)
# one file
sw.scan_inbox()
self.assertEqual([filepath], sw.to_ingest)
# ingest
sw.ingest_all()
self.assertEqual([(sdid, filepath)], self.storage.uploads)
# doesn't upload again
sw.scan_inbox()
sw.ingest_all()
self.assertEqual(1, len(self.storage.uploads))