def test_push(remote_base_path, file_name_1, file_name_2):
with TemporaryDirectory() as test_dir:
local_path_1 = join(test_dir, file_name_1)
local_path_2 = join(test_dir, file_name_2)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# prep files
with open(local_path_1, "w") as file1, open(local_path_2,
"w") as file2:
file1.write('Hello, 1!')
file2.write('Hello, 2!')
# push directory
ticket = TerrainTicket.get([remote_path], mode='write')
irods_commands.push(test_dir, remote_path, ticket)
# check files were pushed to store
files = list_files(remote_path, token)
assert len(files) == 2
assert file_name_1 in files
assert file_name_2 in files
finally:
delete_collection(remote_path, token)
def test_list_directory(remote_base_path):
with TemporaryDirectory() as testdir:
file1_name = 'f1.txt'
file2_name = 'f2.txt'
file1_path = join(testdir, file1_name)
file2_path = join(testdir, file2_name)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file1_path, "w") as file1, open(file2_path,
"w") as file2:
file1.write('Hello, 1!')
file2.write('Hello, 2!')
# upload files
upload_file(file1_path, remote_path, token)
upload_file(file2_path, remote_path, token)
# list files
files = terrain_store.list_dir(remote_path, token)
# check files
assert join(remote_path, file1_name) in files
assert join(remote_path, file2_name) in files
finally:
delete_collection(remote_path, token)
def test_file_exists(remote_base_path):
with TemporaryDirectory() as testdir:
file1_name = 'f1.txt'
file2_name = 'f2.txt'
file1_path = join(testdir, file1_name)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file1_path, "w") as file1:
file1.write('Hello, 1!')
# upload files
upload_file(file1_path, remote_path, token)
# create iRODS session
session = iRODSSession(host='data.cyverse.org',
port=1247,
user='******',
password='',
zone='iplant')
ticket = TerrainTicket.get([join(remote_path, file1_name)], uses=2)
Ticket(session, ticket).supply()
# test remote files exist
assert irods_store.file_exists(join(remote_path, file1_name),
session)
assert not irods_store.file_exists(join(remote_path, file2_name),
session)
finally:
delete_collection(remote_path, token)
def test_file_exists(remote_base_path):
with TemporaryDirectory() as testdir:
file1_name = 'f1.txt'
file2_name = 'f2.txt'
file1_path = join(testdir, file1_name)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file1_path, "w") as file1:
file1.write('Hello, 1!')
# upload files
upload_file(file1_path, remote_path, token)
# test remote files exist
assert terrain_store.file_exists(join(remote_path, file1_name),
token)
assert not terrain_store.file_exists(join(remote_path, file2_name),
token)
finally:
delete_collection(remote_path, token)
def test_upload_directory(remote_base_path):
with TemporaryDirectory() as testdir:
file_name1 = 'f1.txt'
file_name2 = 'f2.txt'
file_path1 = join(testdir, file_name1)
file_path2 = join(testdir, file_name2)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file_path1, "w") as file1, open(file_path2,
"w") as file2:
file1.write('Hello, 1!')
file2.write('Hello, 2!')
# upload directory
terrain_store.push_dir(testdir, remote_path, token)
# check download
files = list_files(remote_path, token)
assert file_name1 in files
assert file_name2 in files
finally:
delete_collection(remote_path, token)
def test_download_directory(remote_base_path):
with TemporaryDirectory() as testdir:
file1_name = 'f1.txt'
file2_name = 'f2.txt'
file1_path = join(testdir, file1_name)
file2_path = join(testdir, file2_name)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file1_path, "w") as file1, open(file2_path,
"w") as file2:
file1.write('Hello, 1!')
file2.write('Hello, 2!')
# upload files
upload_file(file1_path, remote_path, token)
upload_file(file2_path, remote_path, token)
# remove files locally
os.remove(file1_path)
os.remove(file2_path)
# download files
terrain_store.pull_dir(remote_path, testdir, token, ['.txt'])
# check downloads
assert isfile(file1_path)
assert isfile(file2_path)
finally:
delete_collection(remote_path, token)
def test_download_file(remote_base_path):
with TemporaryDirectory() as testdir:
file_name = 'f1.txt'
file_path = join(testdir, file_name)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file_path, "w") as file:
file.write('Hello, 1!')
# upload files
upload_file(file_path, remote_path, token)
# download file
terrain_store.pull_file(join(remote_path, file_name), testdir,
token)
# check download
assert isfile(file_path)
finally:
delete_collection(remote_path, token)
def test_directory_exists(remote_base_path):
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# test remote directories exist
assert terrain_store.dir_exists(remote_path, token)
assert not terrain_store.dir_exists(
join(remote_base_path, "notCollection"), token)
finally:
delete_collection(remote_path, token)
def test_pull_then_run_file_input_and_parameters(remote_base_path,
file_name_1):
with TemporaryDirectory() as testdir:
local_path = join(testdir, file_name_1)
remote_path = join(remote_base_path, "testCollection")
options = {
'workdir': testdir,
'image': "docker://alpine:latest",
'command': 'cat $INPUT > $INPUT.$TAG.output',
'input': {
'path': local_path,
'kind': 'file'
},
'parameters': [{
'key': 'TAG',
'value': message
}]
}
try:
# prep CyVerse collection
create_collection(remote_path, token)
# prep file
with open(local_path, "w") as file1:
file1.write('Hello, 1!')
upload_file(local_path, remote_path, token)
os.remove(local_path)
# pull file to test directory
terrain_commands.pull(remote_path, testdir, cyverse_token=token)
# check file was pulled
check_hello(local_path, 1)
remove(local_path)
# expect 1 container
plantit_cli.runner.dask_commands.run_dask(
options=options,
docker_username=environ.get('DOCKER_USERNAME', None),
docker_password=environ.get('DOCKER_PASSWORD', None))
# check local output file was written
output_1 = f"{local_path}.{message}.output"
check_hello(output_1, 1)
remove(output_1)
finally:
delete_collection(remote_path, token)
def test_upload_directory(remote_base_path):
with TemporaryDirectory() as testdir:
file1_name = 'f1.txt'
file2_name = 'f2.txt'
file1_path = join(testdir, file1_name)
file2_path = join(testdir, file2_name)
coll_name = str(uuid.uuid4())
remote_path = join(remote_base_path, coll_name)
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file1_path, "w") as file1, open(file2_path,
"w") as file2:
file1.write('Hello, 1!')
file2.write('Hello, 2!')
# create iRODS session and get ticket
with iRODSSession(host='data.cyverse.org',
port=1247,
user='******',
password='',
zone='iplant') as session:
ticket = TerrainTicket.get([
join(remote_path, file1_name),
join(remote_path, file2_name)
], 'write', False)
Ticket(session, ticket).supply()
# upload files
irods_store.push_dir(testdir,
remote_path,
session=session,
include_patterns=['.txt'])
# check uploads
coll = session.query(Collection).one()
collection = iRODSCollection(session.collections, coll)
file_names = [o.name for o in collection.data_objects]
print(file_names)
assert file1_name in file_names
assert file2_name in file_names
finally:
delete_collection(remote_path, token)
def test_download_directory(remote_base_path):
with TemporaryDirectory() as testdir:
file1_name = 'f1.txt'
file2_name = 'f2.txt'
file1_path = join(testdir, file1_name)
file2_path = join(testdir, file2_name)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file1_path, "w") as file1, open(file2_path,
"w") as file2:
file1.write('Hello, 1!')
file2.write('Hello, 2!')
# upload files
upload_file(file1_path, remote_path, token)
upload_file(file2_path, remote_path, token)
# remove local files
os.remove(file1_path)
os.remove(file2_path)
# create iRODS session and get ticket
with iRODSSession(host='data.cyverse.org',
port=1247,
user='******',
password='',
zone='iplant') as session:
ticket = TerrainTicket.get([remote_path], 'write', False)
Ticket(session, ticket).supply()
# download files
irods_store.pull_dir(remote_path,
testdir,
session=session,
patterns=['.txt'])
# check downloads
assert isfile(file1_path)
assert isfile(file2_path)
finally:
delete_collection(remote_path, token)
def test_directory_exists(remote_base_path):
remote_path = join(remote_base_path, str(uuid.uuid4()))
with TemporaryDirectory() as testdir:
try:
# prep collection
create_collection(remote_path, token)
# create iRODS session
session = iRODSSession(host='data.cyverse.org',
port=1247,
user='******',
password='',
zone='iplant')
ticket = TerrainTicket.get([remote_path])
Ticket(session, ticket).supply()
# test remote directories exist
assert irods_store.dir_exists(remote_path, session)
assert not irods_store.dir_exists(
join(remote_base_path, "not_a_collection"), session)
finally:
delete_collection(remote_path, token)
def test_pull(remote_base_path, file_name_1, file_name_2):
with TemporaryDirectory() as test_dir:
local_path_1 = join(test_dir, file_name_1)
local_path_2 = join(test_dir, file_name_2)
remote_path = join(remote_base_path, str(uuid.uuid4()))
try:
# prep collection
create_collection(remote_path, token)
# prep files
with open(local_path_1, "w") as file1, open(local_path_2,
"w") as file2:
file1.write('Hello, 1!')
file2.write('Hello, 2!')
# push files to remote directory
upload_file(local_path_1, remote_path, token)
upload_file(local_path_2, remote_path, token)
# remove local files
remove(local_path_1)
remove(local_path_2)
# pull directory
ticket = TerrainTicket.get([remote_path])
irods_commands.pull(remote_path, test_dir, ticket)
# check files were pulled
downloaded_path_1 = join(test_dir, file_name_1)
downloaded_path_2 = join(test_dir, file_name_2)
check_hello(downloaded_path_1, 1)
check_hello(downloaded_path_2, 2)
remove(downloaded_path_1)
remove(downloaded_path_2)
finally:
delete_collection(remote_path, token)
def test_download_file(remote_base_path):
with TemporaryDirectory() as testdir:
file_name = 'f1.txt'
file_path = join(testdir, file_name)
remote_path = join(remote_base_path, str(uuid.uuid4()))
local_path = join(testdir, f"d{file_name}")
try:
# prep collection
create_collection(remote_path, token)
# create files
with open(file_path, "w") as file:
file.write('Hello, 1!')
# upload files
upload_file(file_path, remote_path, token)
# create iRODS session and get ticket
with iRODSSession(host='data.cyverse.org',
port=1247,
user='******',
password='',
zone='iplant') as session:
ticket = TerrainTicket.get([remote_path], 'write', False)
Ticket(session, ticket).supply()
# download file
irods_store.pull_file(join(remote_path, file_name),
local_path,
session=session)
# check download
assert isfile(local_path)
finally:
delete_collection(remote_path, token)