def test_iterate_files_and_folders():
# check we attempt to recurse into the folders
store = Storage({})
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'bye.txt'],
folder_names=['foo'])
top_level_response = FakeResponse(200, json)
second_level_url = ('https://api.osf.io/v2/nodes/9zpcy/files/' +
'osfstorage/foo123/')
json = fake_responses.files_node(
'f3szh', 'osfstorage', file_names=['foo/hello2.txt', 'foo/bye2.txt'])
second_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
elif url == second_level_url:
return second_level_response
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as mock_osf_get:
files = list(store.files)
assert len(files) == 4
for file_ in files:
assert isinstance(file_, File)
assert file_.session == store.session
# check right URLs are called in the right order
expected = [((store._files_url, ), ), ((second_level_url, ), )]
assert mock_osf_get.call_args_list == expected
def mocked_osfcore_get(url):
called.append(url)
if url == store._files_url:
json = fake_responses.files_node('f3szh',
'osfstorage',
folder_names=['foo', 'bar'])
return FakeResponse(200, json)
elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/bar123/':
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['bar.txt'])
return FakeResponse(200, json)
elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/foo123/':
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['foo.txt'],
folder_names=['childfoo'])
return FakeResponse(200, json)
elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/childfoo123/':
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['childfoo.txt'])
return FakeResponse(200, json)
else:
raise ValueError(url)
def fake_get(url, stream):
raw = MagicMock()
src = io.BytesIO(file_content)
raw.read = src.read
res = FakeResponse(200, {})
res.raw = raw
res.headers = {}
return res
def simple_OSFCore_get(url):
if url == 'https://api.osf.io/v2//nodes/f3szh/':
return FakeResponse(200, njson)
elif url == 'https://api.osf.io/v2/nodes/f3szh/files/':
return FakeResponse(200, sjson)
elif url == 'https://api.osf.io/v2//guids/f3szh/':
return FakeResponse(200, {'data': {'type': 'nodes'}})
else:
print(url)
raise ValueError()
def fake_get(url, stream):
raw = MagicMock()
src = io.BytesIO(file_content)
raw.read = src.read
if url == web_url:
raise UnauthorizedException()
else:
res = FakeResponse(200, {})
res.raw = raw
res.headers = {'Content-Length': str(len(file_content))}
return res
def simple_put(url, params={}, data=None):
if url == new_folder_url:
# this is a full fledged Folder response but also works as a
# fake for _WaterButlerFolder
return FakeResponse(
201, {'data': fake_responses._folder('bar12', 'bar')})
elif url == new_file_url:
# we don't do anything with the response, so just make it None
return FakeResponse(201, None)
else:
print(url)
assert False, 'Whoops!'
def test_iterate_files_and_folders():
# check we do not attempt to recurse into the subfolders
store = Folder({})
store._files_url = _files_url
json = fake_responses.files_node('f3szh', 'osfstorage',
file_names=['hello.txt', 'bye.txt'],
folder_names=['bar'])
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
else:
print(url)
raise ValueError()
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as mock_osf_get:
files = list(store.files)
assert len(files) == 2
for file_ in files:
assert isinstance(file_, File)
assert file_.session == store.session
assert file_.name in ('hello.txt', 'bye.txt')
# check we did not try to recurse into subfolders
expected = [((_files_url,),)]
assert mock_osf_get.call_args_list == expected
def test_update_existing_file_fails():
# test we raise an error when we fail to update a file that we think
# exists
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
def simple_OSFCore_put(url, params=None, data=None):
if url == new_file_url:
return FakeResponse(409, None)
elif url.endswith("osfstorage/foo.txt"):
return FakeResponse(200, None)
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node(
'f3szh',
'osfstorage',
# this is the key, none of the files are
# named after the file we are trying to
# update
file_names=['hello.txt', 'bar.txt'])
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
fake_fp = MagicMock()
fake_fp.mode = 'rb'
with patch.object(OSFCore, '_put', side_effect=simple_OSFCore_put):
with patch.object(OSFCore, '_get', side_effect=simple_OSFCore_get):
with pytest.raises(RuntimeError):
store.create_file('foo.txt', fake_fp, update=True)
def test_create_new_zero_length_file():
# check zero length files are special cased
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
store._put = MagicMock(return_value=FakeResponse(201, None))
fake_fp = MagicMock()
fake_fp.mode = 'rb'
if six.PY3:
fake_fp.peek = lambda: ''
if six.PY2:
fake_fp.read = lambda: ''
store.create_file('foo.txt', fake_fp)
store._put.assert_called_once_with(
new_file_url,
# this is the important check in
# this test
data=b'',
params={'name': 'foo.txt'})
assert fake_fp.call_count == 0
def test_remove_file():
f = File({})
f._delete_url = 'http://delete.me/uri'
f._delete = MagicMock(return_value=FakeResponse(204, {'data': {}}))
f.remove()
assert f._delete.called
def test_remove_folder():
folder = Folder({})
folder._delete_url = 'http://delete.me/uri'
folder._delete = MagicMock(return_value=FakeResponse(204, {'data': {}}))
folder.remove()
assert folder._delete.called
def response(url, *args, **kwargs):
data = {
"data": {
"relationships": {
"nodes": {
"links": {
"related": {
"href": user_nodes_url
}
}
}
}
}
}
print(url)
if url == user_me_url:
return FakeResponse(200, data)
else:
return FakeResponse(200, {"data": project_list})
def test_pass_down_session_to_storages(OSFCore_get):
# as previous test but for multiple storages
project = Project({})
project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/'
store_json = fake_responses.storage_node('f3szh')
response = FakeResponse(200, store_json)
OSFCore_get.return_value = response
for store in project.storages:
assert store.session == project.session
def test_invalid_storage(OSFCore_get):
project = Project({})
project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/'
response = FakeResponse(200, fake_responses.storage_node('f3szh'))
OSFCore_get.return_value = response
with pytest.raises(RuntimeError):
project.storage('does-not-exist')
OSFCore_get.assert_called_once_with(
'https://api.osf.io/v2//nodes/f3szh/files/')
def test_create_existing_folder():
folder = Folder({})
new_folder_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/?kind=folder')
folder._new_folder_url = new_folder_url
folder._put = MagicMock(return_value=FakeResponse(409, None))
with pytest.raises(FolderExistsException):
folder.create_folder('foobar')
folder._put.assert_called_once_with(new_folder_url,
params={'name': 'foobar'})
def test_valid_storage(OSFCore_get):
project = Project({})
project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/'
response = FakeResponse(200, fake_responses.storage_node('f3szh'))
OSFCore_get.return_value = response
storage = project.storage('osfstorage')
OSFCore_get.assert_called_once_with(
'https://api.osf.io/v2//nodes/f3szh/files/')
assert isinstance(storage, Storage)
def test_pass_down_session_to_storage(OSFCore_get):
# check that `self.session` is passed to newly created OSFCore instances
project = Project({})
project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/'
store_json = fake_responses.storage_node('f3szh')
response = FakeResponse(200, store_json)
OSFCore_get.return_value = response
store = project.storage()
assert store.session == project.session
def test_remove_file_failed():
f = File({})
f.path = 'some/path'
f._delete_url = 'http://delete.me/uri'
f._delete = MagicMock(return_value=FakeResponse(404, {'data': {}}))
with pytest.raises(RuntimeError) as e:
f.remove()
assert f._delete.called
assert 'Could not delete' in e.value.args[0]
def test_move_file_to_dir():
f = File({})
f._move_url = 'http://move.me/uri'
f._post = MagicMock(return_value=FakeResponse(201, {'data': {}}))
folder = Folder({})
folder.path = 'sample/'
f.move_to('osfclient', folder)
f._post.assert_called_once_with('http://move.me/uri',
json={'action': 'move', 'path': 'sample/'})
def test_move_folder_to_specified_name():
f = Folder({})
f._move_url = 'http://move.me/uri'
f._post = MagicMock(return_value=FakeResponse(201, {'data': {}}))
folder = Folder({})
folder.path = 'sample/'
f.move_to('osfclient', folder, to_foldername='newname')
f._post.assert_called_once_with('http://move.me/uri',
json={'action': 'move', 'path': 'sample/',
'rename': 'newname'})
def test_force_move_folder():
f = Folder({})
f._move_url = 'http://move.me/uri'
f._post = MagicMock(return_value=FakeResponse(201, {'data': {}}))
folder = Folder({})
folder.path = 'sample/'
f.move_to('osfclient', folder, force=True)
f._post.assert_called_once_with('http://move.me/uri',
json={'action': 'move', 'path': 'sample/',
'conflict': 'replace'})
def test_create_new_folder():
folder = Folder({})
new_folder_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/?kind=folder')
folder._new_folder_url = new_folder_url
# use an empty response as we won't do anything with the returned instance
folder._put = MagicMock(return_value=FakeResponse(201, {'data': {}}))
new_folder = folder.create_folder('foobar')
assert isinstance(new_folder, _WaterButlerFolder)
folder._put.assert_called_once_with(new_folder_url,
params={'name': 'foobar'})
def test_move_file_failed():
f = File({})
f.path = 'some/path'
f._move_url = 'http://move.me/uri'
# TODO
f._post = MagicMock(return_value=FakeResponse(204, {'data': {}}))
folder = Folder({})
folder.path = 'sample/'
with pytest.raises(RuntimeError) as e:
f.move_to('osfclient', folder)
assert f._post.called
assert 'Could not move' in e.value.args[0]
def test_iterate_storages(OSFCore_get):
project = Project({})
project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/'
store_json = fake_responses.storage_node('f3szh', ['osfstorage', 'github'])
response = FakeResponse(200, store_json)
OSFCore_get.return_value = response
stores = list(project.storages)
assert len(stores) == 2
for store in stores:
assert isinstance(store, Storage)
OSFCore_get.assert_called_once_with(
'https://api.osf.io/v2//nodes/f3szh/files/')
def test_create_existing_folder_exist_ok():
folder = Folder({})
new_folder_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/?kind=folder')
folder._new_folder_url = new_folder_url
folder._put = MagicMock(return_value=FakeResponse(409, None))
with patch.object(Folder, 'folders',
new_callable=PropertyMock) as mock_folder:
mock_folder.return_value = [MockFolder('foobar'), MockFolder('fudge')]
existing_folder = folder.create_folder('foobar', exist_ok=True)
assert existing_folder.name == 'foobar'
folder._put.assert_called_once_with(new_folder_url,
params={'name': 'foobar'})
def test_update_existing_file_files_match_force_overrides_update():
# test that adding `force=True` and `update=True` forces overwriting of the
# remote file, since `force=True` overrides `update=True`
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
def simple_OSFCore_put(url, params=None, data=None):
if url == new_file_url:
return FakeResponse(409, None)
elif url.endswith("osfstorage/foo.txt"):
return FakeResponse(200, None)
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'foo.txt'])
for i_file in range(2):
json['data'][i_file]['attributes']['extra']['hashes']['md5'] = '0' * 32
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
def simple_checksum(file_path):
return '0' * 32
fake_fp = MagicMock()
fake_fp.mode = 'rb'
with patch.object(OSFCore, '_put',
side_effect=simple_OSFCore_put) as fake_put:
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as fake_get:
with patch('osfclient.models.storage.checksum',
side_effect=simple_checksum):
store.create_file('foo.txt', fake_fp, force=True, update=True)
assert fake_fp.call_count == 0
assert call.peek(1) in fake_fp.mock_calls
# should have made two PUT requests, first attempt at uploading then
# to update the file, even though they match, since force=True overrides
# update=True
assert fake_put.call_count == 2
# should have made one GET request to list files
assert fake_get.call_count == 1
def test_create_new_file():
# create a new file at the top level
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
store._put = MagicMock(return_value=FakeResponse(201, None))
fake_fp = MagicMock()
fake_fp.mode = 'rb'
store.create_file('foo.txt', fake_fp)
store._put.assert_called_once_with(new_file_url,
data=fake_fp,
params={'name': 'foo.txt'})
assert fake_fp.call_count == 0
def test_iterate_files(OSFCore_get):
store = Storage({})
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh', 'osfstorage',
['hello.txt', 'bye.txt'])
response = FakeResponse(200, json)
OSFCore_get.return_value = response
files = list(store.files)
assert len(files) == 2
for file_ in files:
assert isinstance(file_, File)
assert file_.session == store.session
OSFCore_get.assert_called_once_with(
'https://api.osf.io/v2//nodes/f3szh/files/osfstorage')
def test_iterate_folders(OSFCore_get):
store = Folder({})
store._files_url = _files_url
json = fake_responses.files_node('f3szh', 'osfstorage',
folder_names=['foo/bar', 'foo/baz'])
response = FakeResponse(200, json)
OSFCore_get.return_value = response
folders = list(store.folders)
assert len(folders) == 2
for folder in folders:
assert isinstance(folder, Folder)
assert folder.session == store.session
assert folder.name in ('foo/bar', 'foo/baz')
OSFCore_get.assert_called_once_with(
'https://api.osf.io/v2//nodes/f3szh/files/osfstorage/foo123')
def test_update_existing_file_overrides_connection_error():
# successful upload even on connection error if update=True
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
def simple_OSFCore_put(url, params=None, data=None):
if url == new_file_url:
raise ConnectionError
elif url.endswith("osfstorage/foo.txt"):
return FakeResponse(200, None)
def simple_checksum(file_path):
return '0' * 32
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'foo.txt'])
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
fake_fp = MagicMock()
fake_fp.mode = 'rb'
with patch.object(OSFCore, '_put',
side_effect=simple_OSFCore_put) as fake_put:
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as fake_get:
with patch('osfclient.models.storage.checksum',
side_effect=simple_checksum):
store.create_file('foo.txt', fake_fp, update=True)
assert fake_fp.call_count == 0
assert call.peek(1) in fake_fp.mock_calls
# should have made two PUT requests, first attempt at uploading then
# to update the file
assert fake_put.call_count == 2
# should have made one GET request to list files
assert fake_get.call_count == 1
