def download():
# define which file to download
remote_file = Path('/home/xenon/sleep.stdout.txt')
local_file = Path(
'/home/daisycutter/tmp/home/tutorial/xenon/sleep.stdout.txt')
# create the destination file only if the destination path doesn't exist yet
mode = CopyRequest.CREATE
# no need to recurse, we're just uploading a file
recursive = False
# perform the copy/download and wait 1000 ms for the successful or
# otherwise completion of the operation
copy_id = remote_fs.copy(remote_file,
local_fs,
local_file,
mode=mode,
recursive=recursive)
copy_status = remote_fs.wait_until_done(copy_id, timeout=5000)
assert copy_status.done
# rethrow the Exception if we got one
assert copy_status.error_type == CopyStatus.NONE, copy_status.error_message
print('Done downloading.')
def test_copy_local_absolute(xenon_server, tmpdir):
# use the local file system adaptor to create a file system
# representation
with FileSystem.create(adaptor='file') as filesystem:
# create Paths for the source and destination files, using absolute
# paths
dest_file_name = str(tmpdir.join('thefile.bak'))
source_file_name = str(tmpdir.join('thefile.txt'))
with open(source_file_name, 'w') as f:
print("Hello, World!", file=f)
assert os.path.exists(source_file_name)
source_path = Path(source_file_name)
dest_path = Path(dest_file_name)
# create the destination file only if the destination path doesn't
# exist yet; perform the copy and wait 1000 ms for the successful or
# otherwise completion of the operation
copy_id = filesystem.copy(
source_path, filesystem, dest_path,
mode=CopyRequest.CREATE, recursive=False)
timeout_milli_secs = 1000
copy_status = filesystem.wait_until_done(copy_id, timeout_milli_secs)
print("State: ", copy_status.state)
if not copy_status.done:
raise RuntimeError(copy_status.error)
else:
assert os.path.exists(dest_file_name)
def test_files_writing_oop(xenon_server, tmpdir):
with FileSystem.create(adaptor='file') as remotefs:
test_data = [random.randint(0, 255) for i in range(16)]
test_file = str(tmpdir.join('test-writing.txt'))
remotefs.create_file(Path(test_file))
data_stream = iter("{}\n".format(x).encode() for x in test_data)
remotefs.append_to_file(Path(test_file), data_stream)
out_data = [int(line.strip())
for line in open(test_file) if line != '']
assert test_data == out_data
def test_download_remote(xenon_server, tmpdir):
tmpdir = Path(str(tmpdir))
test_data = [random.randint(0, 255) for i in range(16)]
test_file = tmpdir.joinpath('random.txt')
f = open(str(test_file), 'w')
for x in test_data:
print(x, file=f)
f.close()
remote_fs = FileSystem.create(adaptor='file')
# use the local file system adaptor to create a file system
# representation
local_fs = FileSystem.create(adaptor='file')
# define which file to download
remote_file = tmpdir.joinpath('random.txt')
# define what file to download to
local_dir = tmpdir.joinpath('local')
pathlib.Path(str(local_dir)).mkdir()
local_file = local_dir.joinpath(remote_file.name)
# create the destination file only if the destination path doesn't
# exist yet
mode = CopyRequest.CREATE
# no need to recurse, we're just downloading a file
recursive = False
# perform the copy/download and wait 1000 ms for the successful or
# otherwise completion of the operation
copy_id = remote_fs.copy(remote_file,
local_fs,
local_file,
mode=mode,
recursive=recursive)
copy_status = remote_fs.wait_until_done(copy_id, timeout=1000)
if copy_status.error_message:
print(copy_status.error_message)
else:
print('Done')
assert filecmp.cmp(str(local_file), str(remote_file))
# remember to close the FileSystem instances, or use them as
# context managers
remote_fs.close()
local_fs.close()
def test_file_does_not_exist(local_filesystem, tmpdir):
tmpdir = Path(str(tmpdir))
with pytest.raises(NoSuchPathException):
filename = tmpdir / 'this-file-does-not-exist'
result = bytearray()
for chunk in local_filesystem.read_from_file(filename):
result.extend(chunk)
def test_files_reading_oop(xenon_server, tmpdir):
with FileSystem.create(adaptor='file') as remotefs:
test_data = [random.randint(0, 255) for i in range(16)]
test_file = str(tmpdir.join('test-reading.txt'))
f = open(test_file, 'w')
for x in test_data:
print(x, file=f)
f.close()
stream = remotefs.read_from_file(Path(test_file))
out_data = [int(line) for line in read_lines(stream) if line != '']
assert test_data == out_data
from xenon import (
PasswordCredential, FileSystem, CopyRequest, Path, CopyStatus,
Scheduler, JobDescription)
import xenon
xenon.init(log_level='INFO')
tmpdir = Path('.')
location = 'localhost:10022'
#
# step 0: write our script
#
sleep_script = [
"#!/bin/bash",
"echo \"Sleeping for $1 second(s).\"",
"sleep $1"
]
with open(str(tmpdir / 'sleep.sh'), 'w') as f:
for line in sleep_script:
print(line, file=f)
#
# step 1: upload input files
#
# create the local filesystem representation
local_fs = FileSystem.create(adaptor='file')
# the remote system requires credentials, create them here:
xenon.init()
# use the local file system adaptor to create another file system representation
local_fs = FileSystem.create(adaptor='file')
# use the sftp file system adaptor to create a file system
# representation; the remote filesystem requires credentials to log in,
# so we'll have to create those too.
credential = PasswordCredential(username='******', password='******')
remote_fs = FileSystem.create(adaptor='sftp',
location='localhost:10022',
password_credential=credential)
# define which file to upload
local_file = Path('/home/daisycutter/tmp/home/tutorial/xenon/sleep.sh')
remote_file = Path('/home/xenon/sleep.sh')
# create the destination file only if the destination path doesn't exist yet
mode = CopyRequest.CREATE
# no need to recurse, we're just uploading a file
recursive = False
# perform the copy/upload and wait 1000 ms for the successful or
# otherwise completion of the operation
copy_id = local_fs.copy(local_file,
remote_fs,
remote_file,
mode=mode,
recursive=recursive)
def test_posix_file_permissions(local_filesystem, tmpdir):
tmpdir = Path(str(tmpdir))
filename = tmpdir / 'test.dat'
local_filesystem.create_file(filename)
local_filesystem.set_posix_file_permissions(filename, [
PosixFilePermission.OWNER_READ])
def test_path_already_exists_error_2(local_filesystem, tmpdir):
tmpdir = Path(str(tmpdir))
with pytest.raises(PathAlreadyExistsException):
local_filesystem.create_directories(tmpdir / 'test' / '123')
local_filesystem.create_directories(tmpdir / 'test' / '123')
import xenon
from xenon import Path, FileSystem, CopyRequest, CopyStatus
xenon.init()
filesystem = FileSystem.create(adaptor='file')
source_file = Path("/home/daisycutter/tmp/home/tutorial/xenon/thefile.txt")
dest_file = Path("/home/daisycutter/tmp/home/tutorial/xenon/thefile.bak")
# start the copy operation; no recursion, we're just copying a file
copy_id = filesystem.copy(source_file,
filesystem,
dest_file,
mode=CopyRequest.CREATE,
recursive=False)
# wait this many milliseconds for copy operations to complete
WAIT_DURATION = 1000 * 60 * 10
# wait for the copy operation to complete (successfully or otherwise)
copy_status = filesystem.wait_until_done(copy_id, WAIT_DURATION)
assert copy_status.done
# rethrow the Exception if we got one
assert copy_status.error_type == CopyStatus.NONE, copy_status.error_message
filesystem.close()
import xenon
from xenon import Path, FileSystem
xenon.init()
filesystem = FileSystem.create(adaptor='file')
path = Path("/home/tutorial/xenon")
listing = filesystem.list(path, recursive=True)
for entry in listing:
if not entry.path.is_hidden():
print(entry.path)
filesystem.close()
xenon.init()
# use the sftp file system adaptor to create a file system
# representation; the remote filesystem requires credentials to log in,
# so we'll have to create those too.
credential = PasswordCredential(username='******', password='******')
remote_fs = FileSystem.create(adaptor='sftp',
location='localhost:10022',
password_credential=credential)
# use the local file system adaptor to create another file system representation
local_fs = FileSystem.create(adaptor='file')
# define which file to download
remote_file = Path('/home/xenon/sleep.stdout.txt')
local_file = Path('/home/daisycutter/tmp/home/tutorial/xenon/sleep.stdout.txt')
# create the destination file only if the destination path doesn't exist yet
mode = CopyRequest.CREATE
# no need to recurse, we're just uploading a file
recursive = False
# perform the copy/download and wait 1000 ms for the successful or
# otherwise completion of the operation
copy_id = remote_fs.copy(remote_file,
local_fs,
local_file,
mode=mode,
recursive=recursive)
import xenon
from xenon import Path, FileSystem, PasswordCredential
xenon.init()
credential = PasswordCredential(username='******', password='******')
remote_fs = FileSystem.create(adaptor='sftp',
location='localhost:10022',
password_credential=credential)
path = Path("/home/xenon")
listing = remote_fs.list(path, recursive=False)
for entry in listing:
if not entry.path.is_hidden():
print(entry.path)
remote_fs.close()
