def test_workflow(self):
stream_sizes = [2**i for i in range(18, 22)]
# Get expected results from FileDataGenerators
md5_of_files = [
self.__checksum(ReversibleTestFile(s).read()) for s in stream_sizes
]
md5_of_reversed_files = [
self.__checksum(bytes(reversed(ReversibleTestFile(s).read())))
for s in stream_sizes
]
# Get checksum for initial files
workflow = WorkFlow(Meta(MD5, Size), Program("rev"),
Program("tr -d '\n'"), Meta(MD5, Size))
for f in workflow.compose(
TestStream(s, f'{s}', reversible=True) for s in stream_sizes):
f[Stream].read(1)
# Assert that we are not able to retrieve calculated data before
# files have been completely read
with self.assertRaises(FileDataException):
x = f[MD5]
with self.assertRaises(FileDataException):
x = f[Size]
f[Stream].read()
# Assert that checksum created in two different ways are equal
self.assertEqual(f.parent.parent[MD5], md5_of_files.pop(0))
self.assertEqual(f[MD5], md5_of_reversed_files.pop(0))
self.assertEqual(f[Path], str(f[Size]))
# Assert that we've checked all files
self.assertEqual(len(md5_of_files) + len(md5_of_reversed_files), 0)
def test_local(self):
file_names = ('.x.test', '.y.test')
append_to_file_name = '.dat'
try:
workflow = WorkFlow(
Local(process_meta=lambda x: Path(x[Path] + append_to_file_name
)), Meta(Size, MD5))
count = 0
for f in workflow.compose(ByteFile(b'x' * 10, Path(file_names[0])),
ByteFile(b'y' * 20,
Path(file_names[1]))):
content = f[Stream].read()
with open(f[Path], 'rb') as local_file:
count += 1
self.assertEqual(content, local_file.read())
workflow = WorkFlow(Local())
for f in workflow.compose(
LocalFile(file_names[0] + append_to_file_name)):
self.assertEqual(len(f[Stream].read()), 10)
count += 1
self.assertEqual(count, 3)
finally:
for f_n in file_names:
try:
os.remove(f_n)
except:
pass
try:
os.remove(f_n + append_to_file_name)
except:
pass
def test_flush(self):
stream_sizes = [2**i for i in range(18, 22)]
stream_sizes_copy = copy(stream_sizes)
def assert_file_properties(file_stream: File):
_f = file_stream[Stream]
self.assertTrue(_f.readable())
self.assertTrue(_f.writable())
workflow = WorkFlow(Meta(Size), Method(assert_file_properties))
for f in workflow.compose(
TestStream(s, f'{s}', reversible=True)
for s in stream_sizes).flush_iter():
self.assertEqual(f[Size], stream_sizes_copy.pop(0))
def test_local_passthrough(self):
file_names = ('.x.test', '.y.test')
try:
workflow = WorkFlow(Local(pass_through=True), Meta(Size, MD5))
count = 0
for f in workflow.compose(
ByteFile(b'x' * 2**26, Path(file_names[0])),
ByteFile(b'y' * 2**26, Path(file_names[1]))):
content = f[Stream].read()
with open(f[Path], 'rb') as local_file:
self.assertEqual(content, local_file.read())
count += 1
self.assertEqual(count, 2)
finally:
for f_n in file_names:
os.remove(f_n)
def test_process(self):
size = 2**28
signal = False
for file in Meta(Size, MD5).chain(
Program("cat /dev/stdin").chain(TestStream(size, 'xyz'))):
with self.assertRaises(FileDataException):
x = file[Size]
with self.assertRaises(FileDataException):
x = file[MD5]
while file[Stream].read(PIPE_BUFFER_SIZE):
...
self.assertEqual(file[Size], size)
self.assertNotEqual(file[MD5], '')
signal = True
self.assertTrue(signal)
def test_chaining_ftp_server(self):
port = 2121
ftp_server_thread = TestFTPServer(port=port)
ftp_server_thread.start()
files_in = {
f"temp_{size}.testfile": size
for size in (
2 ** x for x in range(20, 24)
)
}
# Flag that is checked to ensure certain parts of the code has been run
run_balance = False
try:
for path, size in files_in.items():
with open(path, 'wb') as f:
f.write(b'x' * size)
gen = FTP().chain(
FTPFile(
path,
host='localhost',
username='******',
password='******',
block_size=PIPE_BUFFER_SIZE,
port=port
) for path in files_in.keys()
)
# Checksum of input
gen = Meta(MD5).chain(
gen
)
# Encrypt and decrypt
gen = Program("gpg --batch --symmetric --passphrase 'X'").chain(
gen
)
# # Checksum of encrypted
gen = Meta(MD5).chain(
gen
)
gen = Program("gpg --batch --decrypt --passphrase 'X'").chain(
gen
)
# # Checksum of decrypted
gen = Meta(MD5).chain(
gen
)
for file_out in gen:
content_out = file_out[Stream].read()
run_balance = False
with open(file_out[Path], 'rb') as f:
source_content = f.read()
md5 = hashlib.md5()
md5.update(source_content)
# Source and output are equal
self.assertEqual(content_out, source_content)
# Source file
self.assertEqual(file_out[MD5, 2], md5.hexdigest())
# Encrypted file
self.assertNotEqual(file_out[MD5, 1], md5.hexdigest())
# Decrypted file
self.assertEqual(file_out[MD5], md5.hexdigest())
run_balance = True
self.assertTrue(run_balance)
except Exception:
raise
finally:
for path in files_in.keys():
try:
os.remove(path)
except:
pass
ftp_server_thread.stop()
self.assertTrue(run_balance)