def mocked_osfcore_get(url):
called.append(url)
if url == store._files_url:
json = fake_responses.files_node('f3szh',
'osfstorage',
folder_names=['foo', 'bar'])
return FakeResponse(200, json)
elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/bar123/':
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['bar.txt'])
return FakeResponse(200, json)
elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/foo123/':
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['foo.txt'],
folder_names=['childfoo'])
return FakeResponse(200, json)
elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/childfoo123/':
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['childfoo.txt'])
return FakeResponse(200, json)
else:
raise ValueError(url)
def test_iterate_files_and_folders():
# check we attempt to recurse into the folders
store = Storage({})
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'bye.txt'],
folder_names=['foo'])
top_level_response = FakeResponse(200, json)
second_level_url = ('https://api.osf.io/v2/nodes/9zpcy/files/' +
'osfstorage/foo123/')
json = fake_responses.files_node(
'f3szh', 'osfstorage', file_names=['foo/hello2.txt', 'foo/bye2.txt'])
second_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
elif url == second_level_url:
return second_level_response
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as mock_osf_get:
files = list(store.files)
assert len(files) == 4
for file_ in files:
assert isinstance(file_, File)
assert file_.session == store.session
# check right URLs are called in the right order
expected = [((store._files_url, ), ), ((second_level_url, ), )]
assert mock_osf_get.call_args_list == expected
def test_update_existing_file_fails():
# test we raise an error when we fail to update a file that we think
# exists
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
def simple_OSFCore_put(url, params=None, data=None):
if url == new_file_url:
return FakeResponse(409, None)
elif url.endswith("osfstorage/foo.txt"):
return FakeResponse(200, None)
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node(
'f3szh',
'osfstorage',
# this is the key, none of the files are
# named after the file we are trying to
# update
file_names=['hello.txt', 'bar.txt'])
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
fake_fp = MagicMock()
fake_fp.mode = 'rb'
with patch.object(OSFCore, '_put', side_effect=simple_OSFCore_put):
with patch.object(OSFCore, '_get', side_effect=simple_OSFCore_get):
with pytest.raises(RuntimeError):
store.create_file('foo.txt', fake_fp, update=True)
def test_iterate_files_and_folders():
# check we do not attempt to recurse into the subfolders
store = Folder({})
store._files_url = _files_url
json = fake_responses.files_node('f3szh', 'osfstorage',
file_names=['hello.txt', 'bye.txt'],
folder_names=['bar'])
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
else:
print(url)
raise ValueError()
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as mock_osf_get:
files = list(store.files)
assert len(files) == 2
for file_ in files:
assert isinstance(file_, File)
assert file_.session == store.session
assert file_.name in ('hello.txt', 'bye.txt')
# check we did not try to recurse into subfolders
expected = [((_files_url,),)]
assert mock_osf_get.call_args_list == expected
def test_long_format_list_with_null(capsys):
args = MockArgs(project='f3szh', long_format=True)
dates = ['null', 'null']
njson = fake_responses._build_node('nodes')
fjson = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'bye.txt'],
file_sizes=['null', 'null'],
file_dates_modified=dates)
sjson = fake_responses.storage_node('f3szh', ['osfstorage'])
def simple_OSFCore_get(url):
if url == 'https://api.osf.io/v2//nodes/f3szh/':
return FakeResponse(200, njson)
elif url == 'https://api.osf.io/v2/nodes/f3szh/files/':
return FakeResponse(200, sjson)
elif url == 'https://api.osf.io/v2/nodes/f3szh/files/osfstorage/':
return FakeResponse(200, fjson)
elif url == 'https://api.osf.io/v2//guids/f3szh/':
return FakeResponse(200, {'data': {'type': 'nodes'}})
else:
print(url)
raise ValueError()
with patch('osfclient.cli.get_localzone',
return_value=tz.tzutc()) as mock_get_localzone:
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as mock_osf_get:
list_(args)
captured = capsys.readouterr()
assert captured.err == ''
expected = ['- - - osfstorage/bye.txt', '- - - osfstorage/hello.txt', '']
assert captured.out.split('\n') == expected
def test_list(capsys):
args = MockArgs(project='f3szh')
njson = fake_responses._build_node('nodes')
rjson = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'bye.txt'],
folder_names=['folder1', 'folder2'])
fjson1 = fake_responses.files_node(
'f3szh', 'osfstorage', file_names=['folder1/folder1content.txt'])
fjson2 = fake_responses.files_node(
'f3szh', 'osfstorage', file_names=['folder2/folder2content.txt'])
sjson = fake_responses.storage_node('f3szh', ['osfstorage'])
def simple_OSFCore_get(url):
if url == 'https://api.osf.io/v2//nodes/f3szh/':
return FakeResponse(200, njson)
elif url == 'https://api.osf.io/v2/nodes/f3szh/files/':
return FakeResponse(200, sjson)
elif url == 'https://api.osf.io/v2/nodes/f3szh/files/osfstorage/':
return FakeResponse(200, rjson)
elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/folder1123/':
return FakeResponse(200, fjson1)
elif url == 'https://api.osf.io/v2/nodes/9zpcy/files/osfstorage/folder2123/':
return FakeResponse(200, fjson2)
elif url == 'https://api.osf.io/v2//guids/f3szh/':
return FakeResponse(200, {'data': {'type': 'nodes'}})
else:
print(url)
raise ValueError()
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as mock_osf_get:
list_(args)
captured = capsys.readouterr()
assert captured.err == ''
assert captured.out.split('\n') == [
'osfstorage/folder2/folder2content.txt',
'osfstorage/folder1/folder1content.txt', 'osfstorage/bye.txt',
'osfstorage/hello.txt', ''
]
def test_update_existing_file_files_match_force_overrides_update():
# test that adding `force=True` and `update=True` forces overwriting of the
# remote file, since `force=True` overrides `update=True`
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
def simple_OSFCore_put(url, params=None, data=None):
if url == new_file_url:
return FakeResponse(409, None)
elif url.endswith("osfstorage/foo.txt"):
return FakeResponse(200, None)
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'foo.txt'])
for i_file in range(2):
json['data'][i_file]['attributes']['extra']['hashes']['md5'] = '0' * 32
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
def simple_checksum(file_path):
return '0' * 32
fake_fp = MagicMock()
fake_fp.mode = 'rb'
with patch.object(OSFCore, '_put',
side_effect=simple_OSFCore_put) as fake_put:
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as fake_get:
with patch('osfclient.models.storage.checksum',
side_effect=simple_checksum):
store.create_file('foo.txt', fake_fp, force=True, update=True)
assert fake_fp.call_count == 0
assert call.peek(1) in fake_fp.mock_calls
# should have made two PUT requests, first attempt at uploading then
# to update the file, even though they match, since force=True overrides
# update=True
assert fake_put.call_count == 2
# should have made one GET request to list files
assert fake_get.call_count == 1
def test_iterate_files(OSFCore_get):
store = Storage({})
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh', 'osfstorage',
['hello.txt', 'bye.txt'])
response = FakeResponse(200, json)
OSFCore_get.return_value = response
files = list(store.files)
assert len(files) == 2
for file_ in files:
assert isinstance(file_, File)
assert file_.session == store.session
OSFCore_get.assert_called_once_with(
'https://api.osf.io/v2//nodes/f3szh/files/osfstorage')
def test_iterate_folders(OSFCore_get):
store = Folder({})
store._files_url = _files_url
json = fake_responses.files_node('f3szh', 'osfstorage',
folder_names=['foo/bar', 'foo/baz'])
response = FakeResponse(200, json)
OSFCore_get.return_value = response
folders = list(store.folders)
assert len(folders) == 2
for folder in folders:
assert isinstance(folder, Folder)
assert folder.session == store.session
assert folder.name in ('foo/bar', 'foo/baz')
OSFCore_get.assert_called_once_with(
'https://api.osf.io/v2//nodes/f3szh/files/osfstorage/foo123')
def test_update_existing_file_overrides_connection_error():
# successful upload even on connection error if update=True
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
def simple_OSFCore_put(url, params=None, data=None):
if url == new_file_url:
raise ConnectionError
elif url.endswith("osfstorage/foo.txt"):
return FakeResponse(200, None)
def simple_checksum(file_path):
return '0' * 32
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'foo.txt'])
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
fake_fp = MagicMock()
fake_fp.mode = 'rb'
with patch.object(OSFCore, '_put',
side_effect=simple_OSFCore_put) as fake_put:
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as fake_get:
with patch('osfclient.models.storage.checksum',
side_effect=simple_checksum):
store.create_file('foo.txt', fake_fp, update=True)
assert fake_fp.call_count == 0
assert call.peek(1) in fake_fp.mock_calls
# should have made two PUT requests, first attempt at uploading then
# to update the file
assert fake_put.call_count == 2
# should have made one GET request to list files
assert fake_get.call_count == 1
def test_force_existing_file():
# test that adding `force=True` lets you overwrite existing remote files
new_file_url = ('https://files.osf.io/v1/resources/9zpcy/providers/' +
'osfstorage/foo123/')
store = Storage({})
store._new_file_url = new_file_url
def simple_OSFCore_put(url, params=None, data=None):
if url == new_file_url:
return FakeResponse(409, None)
elif url.endswith("osfstorage/foo.txt"):
return FakeResponse(200, None)
store._files_url = 'https://api.osf.io/v2//nodes/f3szh/files/osfstorage'
json = fake_responses.files_node('f3szh',
'osfstorage',
file_names=['hello.txt', 'foo.txt'])
top_level_response = FakeResponse(200, json)
def simple_OSFCore_get(url):
if url == store._files_url:
return top_level_response
fake_fp = MagicMock()
fake_fp.mode = 'rb'
with patch.object(OSFCore, '_put',
side_effect=simple_OSFCore_put) as fake_put:
with patch.object(OSFCore, '_get',
side_effect=simple_OSFCore_get) as fake_get:
store.create_file('foo.txt', fake_fp, force=True)
assert fake_fp.call_count == 0
assert call.peek(1) in fake_fp.mock_calls
# should have made two PUT requests, first attempt at uploading then
# to update the file
assert fake_put.call_count == 2
# should have made one GET request to list files
assert fake_get.call_count == 1
