def test_is_uri(bad_uris):
for scheme in DataUri.SCHEMES:
id = '123456'
uri = '{0}:{1}'.format(scheme, id).title()
assert DataUri.is_uri(uri) is True
for bad_uri in bad_uris:
assert DataUri.is_uri(bad_uri) is False
def test_it_pulls_a_file_not_matching_the_data_structure(
mk_kiproject, syn_non_data, syn_client):
kiproject = mk_kiproject()
syn_parent, syn_folders, syn_files = syn_non_data
# Pull the root files
for syn_file in syn_files:
syn_file_uri = DataUri('syn', syn_file.id).uri
data_type = sample(kiproject.data_types, 1)[0]
# Add the file to the KiProject
ki_project_resource = kiproject.data_add(syn_file_uri,
data_type=data_type)
# Pull the file.
resource = kiproject.data_pull(ki_project_resource.remote_uri)
assert resource
# Check that file/folders exist locally
expected_local_path = os.path.join(kiproject.local_path,
data_type.rel_path, syn_file.name)
assert resource.abs_path == expected_local_path
assert os.path.exists(expected_local_path) is True
# Reset the local dirs/files
for resource in kiproject.find_project_resources_by(root_id=None):
kiproject.data_remove(resource)
SysPath(resource.abs_path).delete()
# Pull some child files
for syn_folder in syn_folders:
syn_child_files = syn_client.getChildren(parent=syn_folder,
includeTypes=['file'])
for syn_child_file in syn_child_files:
syn_file_uri = DataUri('syn', syn_child_file.get('id')).uri
data_type = sample(kiproject.data_types, 1)[0]
# Add the file to the KiProject
ki_project_resource = kiproject.data_add(syn_file_uri,
data_type=data_type)
# Pull the file.
resource = kiproject.data_pull(ki_project_resource.remote_uri)
assert resource
# Check that file/folders exist locally
expected_local_path = os.path.join(kiproject.local_path,
data_type.rel_path,
syn_folder.name,
syn_child_file.get('name'))
assert resource.abs_path == expected_local_path
assert os.path.exists(expected_local_path) is True
def test_data_pull_non_data_folder(syn_test_helper, mk_tempfile,
mk_uniq_string, mk_kiproject):
syn_project = syn_test_helper.create_project()
syn_folder1 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder1', parent=syn_project))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder1))
syn_folder2 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder2', parent=syn_folder1))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder2))
syn_folder3 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder3', parent=syn_folder2))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder3))
syn_folder4 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder4', parent=syn_folder3))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder4))
syn_folder5 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder5', parent=syn_folder4))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder5))
kiproject = mk_kiproject()
kiproject.data_add(DataUri('syn', syn_folder1.id).uri,
data_type=kiproject.data_types[0])
kiproject.data_pull()
def test_it_adds_a_remote_data_structure_folder(mk_kiproject, syn_data):
kiproject = mk_kiproject()
syn_project, syn_folders, syn_files = syn_data
for syn_folder in syn_folders:
ki_project_resource = kiproject.data_add(
DataUri('syn', syn_folder.id).uri)
assert ki_project_resource
def test_it_creates_a_new_remote_project(mk_mock_kiproject_input, mk_tempdir,
syn_test_helper):
mk_mock_kiproject_input(create_remote_project_or_existing='c')
kiproject = KiProject(mk_tempdir())
syn_project = syn_test_helper.client().get(
DataUri.parse(kiproject.project_uri).id)
syn_test_helper.dispose_of(syn_project)
def test_it_adds_a_remote_non_data_structure_file(mk_kiproject, syn_non_data):
kiproject = mk_kiproject()
syn_parent, syn_folders, syn_files = syn_non_data
for syn_file in syn_files:
syn_file_uri = DataUri('syn', syn_file.id).uri
ki_project_resource = kiproject.data_add(
syn_file_uri, data_type=kiproject.data_types[0])
assert ki_project_resource
def test_it_pulls_a_folder_matching_the_data_structure(mk_kiproject, syn_data):
kiproject = mk_kiproject()
syn_project, syn_folders, syn_files = syn_data
for syn_folder in syn_folders:
syn_folder_uri = DataUri('syn', syn_folder.id).uri
# Add the folder to the KiProject
ki_project_resource = kiproject.data_add(syn_folder_uri)
# Pull the folder.
resource = kiproject.data_pull(ki_project_resource.remote_uri)
assert resource
def test___init__(bad_uris):
for scheme in DataUri.SCHEMES:
id = '123456'
uri = '{0}:{1}'.format(scheme, id).title()
duri = DataUri.parse(uri)
assert duri.scheme == scheme
assert duri.id == id
assert duri.uri == uri.lower()
if scheme == 'syn':
assert isinstance(duri.data_adapter(), SynapseAdapter)
with pytest.raises(InvalidDataUriError) as ex:
DataUri.parse(None)
assert str(ex.value) == 'uri cannot be blank.'
for bad_uri in bad_uris:
with pytest.raises(InvalidDataUriError) as ex:
DataUri.parse(bad_uri)
assert ('Invalid URI format, cannot parse:' in str(ex.value)
or str(ex.value) == 'uri cannot be blank.'
or 'Invalid URI scheme:' in str(ex.value)
or str(ex.value) == 'URI ID must be provided.')
def test_it_does_not_push_a_file_unless_the_local_file_changed(
mk_kiproject, mk_syn_files, syn_client, mocker):
kiproject = mk_kiproject()
# Get the Synapse project for the KiProject
syn_project = syn_client.get(DataUri.parse(kiproject.project_uri).id)
syn_data_folder = syn_client.store(
synapseclient.Folder(name='data', parent=syn_project))
syn_core_folder = syn_client.store(
synapseclient.Folder(name='core', parent=syn_data_folder))
# Create a Synapse file to add/pull/push
syn_file = mk_syn_files(syn_core_folder, file_num=1, versions=1,
suffix='')[0]
syn_file_uri = DataUri('syn', syn_file.id).uri
kiproject.data_add(syn_file_uri, data_type=kiproject.data_types[0])
kiproject.data_pull()
# The file exists in the Synapse project and has been pulled locally.
# Pushing again should NOT upload the file again.
mocker.spy(synapseclient.client, 'upload_file_handle')
kiproject.data_push(syn_file_uri)
def test_it_pushes_a_file_to_a_different_remote_project(
syn_client, mk_kiproject, mk_syn_project, mk_syn_files, write_file,
read_file):
kiproject = mk_kiproject()
other_syn_project = mk_syn_project()
syn_file = mk_syn_files(other_syn_project,
file_num=1,
versions=1,
suffix='')[0]
syn_file_uri = DataUri('syn', syn_file.id).uri
kiproject.data_add(syn_file_uri, data_type=kiproject.data_types[0])
resource = kiproject.data_pull(syn_file_uri)
new_file_contents = str(uuid.uuid4())
write_file(resource.abs_path, new_file_contents)
kiproject.data_push(syn_file_uri)
updated_syn_file = syn_client.get(syn_file.id, downloadFile=True)
assert read_file(updated_syn_file.path) == new_file_contents
def syn_project_uri(syn_project):
return DataUri(SynapseAdapter.DATA_URI_SCHEME, syn_project.id).uri
def _mk(scheme='syn'):
return DataUri(scheme, '{0}{1}'.format(scheme, mk_uniq_integer())).uri
def _sdo(kiproject):
syn_project = syn_test_helper.client().get(
DataUri.parse(kiproject.project_uri).id)
syn_test_helper.dispose_of(syn_project)
return syn_project
def test_it_tests_the_workflow(mk_kiproject, mk_local_data_dir, mk_uniq_string,
syn_data, syn_non_data):
kiproject = mk_kiproject()
local_data_folders, local_data_files = mk_local_data_dir(kiproject)
###########################################################################
# Add/Push Local Files and Folders
###########################################################################
# Files
for local_file in local_data_files:
resource = kiproject.data_add(local_file)
push_result = kiproject.data_push(resource)
assert push_result == resource
# Re-add it
resource_count = len(kiproject.resources)
kiproject.data_add(local_file)
assert len(kiproject.resources) == resource_count
# Re-add with a change
new_name = mk_uniq_string()
resource = kiproject.data_add(local_file, name=new_name, version='1')
assert len(kiproject.resources) == resource_count
assert resource.version == '1'
assert resource.name == new_name
resource = kiproject.data_add(local_file)
assert len(kiproject.resources) == resource_count
assert resource.version is None
assert resource.name == os.path.basename(local_file)
# Folders
for local_folder in local_data_folders:
resource = kiproject.data_add(local_folder)
push_result = kiproject.data_push(resource)
assert push_result == resource
###########################################################################
# Add/Pull Remote Data Files and Folders
###########################################################################
_, syn_data_folders, syn_data_files = syn_data
# Files
for syn_file in syn_data_files:
remote_uri = DataUri('syn', syn_file.id).uri
resource = kiproject.data_add(remote_uri)
pull_result = kiproject.data_pull(remote_uri)
assert pull_result == resource
# Lock the version
resource = kiproject.data_change(remote_uri, version='2')
assert resource.version == '2'
kiproject.data_pull(remote_uri)
# Folders
for syn_folder in syn_data_folders:
remote_uri = DataUri('syn', syn_folder.id).uri
resource = kiproject.data_add(remote_uri)
pull_result = kiproject.data_pull(remote_uri)
assert pull_result == resource
###########################################################################
# Add/Pull Remote non-Data Files and Folders
###########################################################################
_, syn_non_data_folders, syn_non_data_files = syn_non_data
# Files
for syn_non_data_file in syn_non_data_files:
remote_uri = DataUri('syn', syn_non_data_file.id).uri
resource = kiproject.data_add(remote_uri,
data_type=kiproject.data_types[0])
pull_result = kiproject.data_pull(remote_uri)
assert pull_result == resource
# Folders
for syn_non_data_folder in syn_non_data_folders:
remote_uri = DataUri('syn', syn_non_data_folder.id).uri
resource = kiproject.data_add(remote_uri,
data_type=kiproject.data_types[0])
pull_result = kiproject.data_pull(remote_uri)
assert pull_result == resource
###########################################################################
# Push/Pull Everything
###########################################################################
kiproject.data_push()
kiproject.data_pull()
###########################################################################
# data_list
###########################################################################
kiproject.data_list()
kiproject.data_list(all=True)
###########################################################################
# data_remove
###########################################################################
for resource in kiproject.resources.copy():
if resource.root_id:
continue
kiproject.data_remove(resource.remote_uri or resource.abs_path)
assert len(kiproject.resources) == 0
###########################################################################
# data_ignore
###########################################################################
ignore_pattern = '*.png'
assert ignore_pattern not in kiproject.data_ignores
kiproject.add_data_ignore(ignore_pattern)
assert ignore_pattern in kiproject.data_ignores
kiproject.remove_data_ignore(ignore_pattern)
assert ignore_pattern not in kiproject.data_ignores
