def test_iterate_files_and_folders(): # check we attempt to recurse into the folders store = Storage({}) store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage' json = fake_responses.files_node('f3szh', 'osfstorage', file_names=['hello.txt', 'bye.txt'], folder_names=['foo']) top_level_response = FakeResponse(200, json) second_level_url = ('https://api.osf.io/v2/nodes/9zpcy/files/' + 'osfstorage/foo123/') json = fake_responses.files_node( 'f3szh', 'osfstorage', file_names=['foo/hello2.txt', 'foo/bye2.txt']) second_level_response = FakeResponse(200, json) def simple_OSFCore_get(url): if url == store._files_url: return top_level_response elif url == second_level_url: return second_level_response with patch.object(OSFCore, '_get', side_effect=simple_OSFCore_get) as mock_osf_get: files = list(store.files) assert len(files) == 4 for file_ in files: assert isinstance(file_, File) assert file_.session == store.session # check right URLs are called in the right order expected = [((store._files_url, ), ), ((second_level_url, ), )] assert mock_osf_get.call_args_list == expected
def mocked_osfcore_get(url): called.append(url) if url == store._files_url: json = fake_responses.files_node('f3szh', 'osfstorage', folder_names=['foo', 'bar']) return FakeResponse(200, json) elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/bar123/': json = fake_responses.files_node('f3szh', 'osfstorage', file_names=['bar.txt']) return FakeResponse(200, json) elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/foo123/': json = fake_responses.files_node('f3szh', 'osfstorage', file_names=['foo.txt'], folder_names=['childfoo']) return FakeResponse(200, json) elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/childfoo123/': json = fake_responses.files_node('f3szh', 'osfstorage', file_names=['childfoo.txt']) return FakeResponse(200, json) else: raise ValueError(url)
def fake_get(url, stream): raw = MagicMock() src = io.BytesIO(file_content) raw.read = src.read res = FakeResponse(200, {}) res.raw = raw res.headers = {} return res
def simple_OSFCore_get(url): if url == 'https://api.osf.io/v2//nodes/f3szh/': return FakeResponse(200, njson) elif url == 'https://api.osf.io/v2/nodes/f3szh/files/': return FakeResponse(200, sjson) elif url == 'https://api.osf.io/v2//guids/f3szh/': return FakeResponse(200, {'data': {'type': 'nodes'}}) else: print(url) raise ValueError()
def fake_get(url, stream): raw = MagicMock() src = io.BytesIO(file_content) raw.read = src.read if url == web_url: raise UnauthorizedException() else: res = FakeResponse(200, {}) res.raw = raw res.headers = {'Content-Length': str(len(file_content))} return res
def simple_put(url, params={}, data=None): if url == new_folder_url: # this is a full fledged Folder response but also works as a # fake for _WaterButlerFolder return FakeResponse( 201, {'data': fake_responses._folder('bar12', 'bar')}) elif url == new_file_url: # we don't do anything with the response, so just make it None return FakeResponse(201, None) else: print(url) assert False, 'Whoops!'
def test_iterate_files_and_folders(): # check we do not attempt to recurse into the subfolders store = Folder({}) store._files_url = _files_url json = fake_responses.files_node('f3szh', 'osfstorage', file_names=['hello.txt', 'bye.txt'], folder_names=['bar']) top_level_response = FakeResponse(200, json) def simple_OSFCore_get(url): if url == store._files_url: return top_level_response else: print(url) raise ValueError() with patch.object(OSFCore, '_get', side_effect=simple_OSFCore_get) as mock_osf_get: files = list(store.files) assert len(files) == 2 for file_ in files: assert isinstance(file_, File) assert file_.session == store.session assert file_.name in ('hello.txt', 'bye.txt') # check we did not try to recurse into subfolders expected = [((_files_url,),)] assert mock_osf_get.call_args_list == expected
def test_update_existing_file_fails(): # test we raise an error when we fail to update a file that we think # exists new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' + 'osfstorage/foo123/') store = Storage({}) store._new_file_url = new_file_url def simple_OSFCore_put(url, params=None, data=None): if url == new_file_url: return FakeResponse(409, None) elif url.endswith("osfstorage/foo.txt"): return FakeResponse(200, None) store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage' json = fake_responses.files_node( 'f3szh', 'osfstorage', # this is the key, none of the files are # named after the file we are trying to # update file_names=['hello.txt', 'bar.txt']) top_level_response = FakeResponse(200, json) def simple_OSFCore_get(url): if url == store._files_url: return top_level_response fake_fp = MagicMock() fake_fp.mode = 'rb' with patch.object(OSFCore, '_put', side_effect=simple_OSFCore_put): with patch.object(OSFCore, '_get', side_effect=simple_OSFCore_get): with pytest.raises(RuntimeError): store.create_file('foo.txt', fake_fp, update=True)
def test_create_new_zero_length_file(): # check zero length files are special cased new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' + 'osfstorage/foo123/') store = Storage({}) store._new_file_url = new_file_url store._put = MagicMock(return_value=FakeResponse(201, None)) fake_fp = MagicMock() fake_fp.mode = 'rb' if six.PY3: fake_fp.peek = lambda: '' if six.PY2: fake_fp.read = lambda: '' store.create_file('foo.txt', fake_fp) store._put.assert_called_once_with( new_file_url, # this is the important check in # this test data=b'', params={'name': 'foo.txt'}) assert fake_fp.call_count == 0
def test_remove_file(): f = File({}) f._delete_url = 'http://delete.me/uri' f._delete = MagicMock(return_value=FakeResponse(204, {'data': {}})) f.remove() assert f._delete.called
def test_remove_folder(): folder = Folder({}) folder._delete_url = 'http://delete.me/uri' folder._delete = MagicMock(return_value=FakeResponse(204, {'data': {}})) folder.remove() assert folder._delete.called
def response(url, *args, **kwargs): data = { "data": { "relationships": { "nodes": { "links": { "related": { "href": user_nodes_url } } } } } } print(url) if url == user_me_url: return FakeResponse(200, data) else: return FakeResponse(200, {"data": project_list})
def test_pass_down_session_to_storages(OSFCore_get): # as previous test but for multiple storages project = Project({}) project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/' store_json = fake_responses.storage_node('f3szh') response = FakeResponse(200, store_json) OSFCore_get.return_value = response for store in project.storages: assert store.session == project.session
def test_invalid_storage(OSFCore_get): project = Project({}) project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/' response = FakeResponse(200, fake_responses.storage_node('f3szh')) OSFCore_get.return_value = response with pytest.raises(RuntimeError): project.storage('does-not-exist') OSFCore_get.assert_called_once_with( 'https://api.osf.io/v2//nodes/f3szh/files/')
def test_create_existing_folder(): folder = Folder({}) new_folder_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' + 'osfstorage/foo123/?kind=folder') folder._new_folder_url = new_folder_url folder._put = MagicMock(return_value=FakeResponse(409, None)) with pytest.raises(FolderExistsException): folder.create_folder('foobar') folder._put.assert_called_once_with(new_folder_url, params={'name': 'foobar'})
def test_valid_storage(OSFCore_get): project = Project({}) project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/' response = FakeResponse(200, fake_responses.storage_node('f3szh')) OSFCore_get.return_value = response storage = project.storage('osfstorage') OSFCore_get.assert_called_once_with( 'https://api.osf.io/v2//nodes/f3szh/files/') assert isinstance(storage, Storage)
def test_pass_down_session_to_storage(OSFCore_get): # check that `self.session` is passed to newly created OSFCore instances project = Project({}) project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/' store_json = fake_responses.storage_node('f3szh') response = FakeResponse(200, store_json) OSFCore_get.return_value = response store = project.storage() assert store.session == project.session
def test_remove_file_failed(): f = File({}) f.path = 'some/path' f._delete_url = 'http://delete.me/uri' f._delete = MagicMock(return_value=FakeResponse(404, {'data': {}})) with pytest.raises(RuntimeError) as e: f.remove() assert f._delete.called assert 'Could not delete' in e.value.args[0]
def test_move_file_to_dir(): f = File({}) f._move_url = 'http://move.me/uri' f._post = MagicMock(return_value=FakeResponse(201, {'data': {}})) folder = Folder({}) folder.path = 'sample/' f.move_to('osfclient', folder) f._post.assert_called_once_with('http://move.me/uri', json={'action': 'move', 'path': 'sample/'})
def test_move_folder_to_specified_name(): f = Folder({}) f._move_url = 'http://move.me/uri' f._post = MagicMock(return_value=FakeResponse(201, {'data': {}})) folder = Folder({}) folder.path = 'sample/' f.move_to('osfclient', folder, to_foldername='newname') f._post.assert_called_once_with('http://move.me/uri', json={'action': 'move', 'path': 'sample/', 'rename': 'newname'})
def test_force_move_folder(): f = Folder({}) f._move_url = 'http://move.me/uri' f._post = MagicMock(return_value=FakeResponse(201, {'data': {}})) folder = Folder({}) folder.path = 'sample/' f.move_to('osfclient', folder, force=True) f._post.assert_called_once_with('http://move.me/uri', json={'action': 'move', 'path': 'sample/', 'conflict': 'replace'})
def test_create_new_folder(): folder = Folder({}) new_folder_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' + 'osfstorage/foo123/?kind=folder') folder._new_folder_url = new_folder_url # use an empty response as we won't do anything with the returned instance folder._put = MagicMock(return_value=FakeResponse(201, {'data': {}})) new_folder = folder.create_folder('foobar') assert isinstance(new_folder, _WaterButlerFolder) folder._put.assert_called_once_with(new_folder_url, params={'name': 'foobar'})
def test_move_file_failed(): f = File({}) f.path = 'some/path' f._move_url = 'http://move.me/uri' # TODO f._post = MagicMock(return_value=FakeResponse(204, {'data': {}})) folder = Folder({}) folder.path = 'sample/' with pytest.raises(RuntimeError) as e: f.move_to('osfclient', folder) assert f._post.called assert 'Could not move' in e.value.args[0]
def test_iterate_storages(OSFCore_get): project = Project({}) project._storages_url = 'https://api.osf.io/v2//nodes/f3szh/files/' store_json = fake_responses.storage_node('f3szh', ['osfstorage', 'github']) response = FakeResponse(200, store_json) OSFCore_get.return_value = response stores = list(project.storages) assert len(stores) == 2 for store in stores: assert isinstance(store, Storage) OSFCore_get.assert_called_once_with( 'https://api.osf.io/v2//nodes/f3szh/files/')
def test_create_existing_folder_exist_ok(): folder = Folder({}) new_folder_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' + 'osfstorage/foo123/?kind=folder') folder._new_folder_url = new_folder_url folder._put = MagicMock(return_value=FakeResponse(409, None)) with patch.object(Folder, 'folders', new_callable=PropertyMock) as mock_folder: mock_folder.return_value = [MockFolder('foobar'), MockFolder('fudge')] existing_folder = folder.create_folder('foobar', exist_ok=True) assert existing_folder.name == 'foobar' folder._put.assert_called_once_with(new_folder_url, params={'name': 'foobar'})
def test_update_existing_file_files_match_force_overrides_update(): # test that adding `force=True` and `update=True` forces overwriting of the # remote file, since `force=True` overrides `update=True` new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' + 'osfstorage/foo123/') store = Storage({}) store._new_file_url = new_file_url def simple_OSFCore_put(url, params=None, data=None): if url == new_file_url: return FakeResponse(409, None) elif url.endswith("osfstorage/foo.txt"): return FakeResponse(200, None) store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage' json = fake_responses.files_node('f3szh', 'osfstorage', file_names=['hello.txt', 'foo.txt']) for i_file in range(2): json['data'][i_file]['attributes']['extra']['hashes']['md5'] = '0' * 32 top_level_response = FakeResponse(200, json) def simple_OSFCore_get(url): if url == store._files_url: return top_level_response def simple_checksum(file_path): return '0' * 32 fake_fp = MagicMock() fake_fp.mode = 'rb' with patch.object(OSFCore, '_put', side_effect=simple_OSFCore_put) as fake_put: with patch.object(OSFCore, '_get', side_effect=simple_OSFCore_get) as fake_get: with patch('osfclient.models.storage.checksum', side_effect=simple_checksum): store.create_file('foo.txt', fake_fp, force=True, update=True) assert fake_fp.call_count == 0 assert call.peek(1) in fake_fp.mock_calls # should have made two PUT requests, first attempt at uploading then # to update the file, even though they match, since force=True overrides # update=True assert fake_put.call_count == 2 # should have made one GET request to list files assert fake_get.call_count == 1
def test_create_new_file(): # create a new file at the top level new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' + 'osfstorage/foo123/') store = Storage({}) store._new_file_url = new_file_url store._put = MagicMock(return_value=FakeResponse(201, None)) fake_fp = MagicMock() fake_fp.mode = 'rb' store.create_file('foo.txt', fake_fp) store._put.assert_called_once_with(new_file_url, data=fake_fp, params={'name': 'foo.txt'}) assert fake_fp.call_count == 0
def test_iterate_files(OSFCore_get): store = Storage({}) store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage' json = fake_responses.files_node('f3szh', 'osfstorage', ['hello.txt', 'bye.txt']) response = FakeResponse(200, json) OSFCore_get.return_value = response files = list(store.files) assert len(files) == 2 for file_ in files: assert isinstance(file_, File) assert file_.session == store.session OSFCore_get.assert_called_once_with( 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage')
def test_iterate_folders(OSFCore_get): store = Folder({}) store._files_url = _files_url json = fake_responses.files_node('f3szh', 'osfstorage', folder_names=['foo/bar', 'foo/baz']) response = FakeResponse(200, json) OSFCore_get.return_value = response folders = list(store.folders) assert len(folders) == 2 for folder in folders: assert isinstance(folder, Folder) assert folder.session == store.session assert folder.name in ('foo/bar', 'foo/baz') OSFCore_get.assert_called_once_with( 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage/foo123')
def test_update_existing_file_overrides_connection_error(): # successful upload even on connection error if update=True new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' + 'osfstorage/foo123/') store = Storage({}) store._new_file_url = new_file_url def simple_OSFCore_put(url, params=None, data=None): if url == new_file_url: raise ConnectionError elif url.endswith("osfstorage/foo.txt"): return FakeResponse(200, None) def simple_checksum(file_path): return '0' * 32 store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage' json = fake_responses.files_node('f3szh', 'osfstorage', file_names=['hello.txt', 'foo.txt']) top_level_response = FakeResponse(200, json) def simple_OSFCore_get(url): if url == store._files_url: return top_level_response fake_fp = MagicMock() fake_fp.mode = 'rb' with patch.object(OSFCore, '_put', side_effect=simple_OSFCore_put) as fake_put: with patch.object(OSFCore, '_get', side_effect=simple_OSFCore_get) as fake_get: with patch('osfclient.models.storage.checksum', side_effect=simple_checksum): store.create_file('foo.txt', fake_fp, update=True) assert fake_fp.call_count == 0 assert call.peek(1) in fake_fp.mock_calls # should have made two PUT requests, first attempt at uploading then # to update the file assert fake_put.call_count == 2 # should have made one GET request to list files assert fake_get.call_count == 1