def test_intra_makes_callback(self):
self.source_provider.copy = utils.MockCoroutine(
return_value=(utils.MockFileMetadata(), False))
yield self.http_client.fetch(self.get_url('/ops/copy'),
method='POST',
body=json.dumps(self.payload()))
self.mock_send_hook.assert_called_once_with('copy',
utils.MockFileMetadata())
def test__return_values(self, providers, bundles, callback, src_path,
dest_path):
src, dest = providers
src_bundle, dest_bundle = bundles
metadata = test_utils.MockFileMetadata()
src.copy.return_value = (metadata, False)
dt = datetime.datetime.utcfromtimestamp(60)
with freezegun.freeze_time(dt):
ret1, ret2 = copy.copy(cp.deepcopy(src_bundle),
cp.deepcopy(dest_bundle), 'Test.com',
{'auth': {
'user': '******'
}})
assert (ret1, ret2) == (metadata, False)
callback.assert_called_once_with(
'PUT', 'Test.com', {
'errors': [],
'action': 'copy',
'source': {
'path': src_path.path,
'name': src_path.name,
'materialized': str(src_path),
'provider': src.NAME,
},
'destination': metadata.serialized(),
'auth': {
'user': '******'
},
'time': 120,
'email': False
})
def test_file_json_api_serialize(self):
file_metadata = utils.MockFileMetadata()
serialized = file_metadata.json_api_serialized('n0d3z')
link_suffix = '/v1/resources/n0d3z/providers/mock/Foo.name'
etag = hashlib.sha256('{}::{}'.format(
'mock', 'etag').encode('utf-8')).hexdigest()
assert serialized['id'] == 'mock/Foo.name'
assert serialized['type'] == 'files'
assert serialized['attributes'] == {
'extra': {},
'kind': 'file',
'name': 'Foo.name',
'path': '/Foo.name',
'provider': 'mock',
'materialized': '/Foo.name',
'etag': etag,
'contentType': 'application/octet-stream',
'modified': 'never',
'size': 1337,
}
assert 'new_folder' not in serialized['links']
assert serialized['links']['move'].endswith(link_suffix)
assert serialized['links']['upload'].endswith(link_suffix +
'?kind=file')
assert serialized['links']['download'].endswith(link_suffix)
assert serialized['links']['delete'].endswith(link_suffix)
def test_return_values(self, event_loop, providers, bundles, callback,
src_path, dest_path, mock_time):
src, dest = providers
src_bundle, dest_bundle = bundles
metadata = test_utils.MockFileMetadata()
src.copy.return_value = (metadata, False)
ret1, ret2 = copy.copy(cp.deepcopy(src_bundle),
cp.deepcopy(dest_bundle), 'Test.com',
{'auth': {
'user': '******'
}})
assert (ret1, ret2) == (metadata, False)
callback.assert_called_once_with(
'PUT', 'Test.com', {
'errors': [],
'action': 'copy',
'source': {
'path': '/' + src_path.raw_path,
'name': src_path.name,
'materialized': str(src_path),
'provider': src.NAME,
'kind': 'file',
},
'destination': metadata.serialized(),
'auth': {
'user': '******'
},
'time': FAKE_TIME + 60,
'email': False
})
async def test_upload_fails(self, monkeypatch, provider_and_mock,
file_stream, upload_response, mock_time):
self.patch_tasks(monkeypatch)
provider, inner_provider = provider_and_mock
path = WaterButlerPath('/{}'.format(upload_response['data']['name']),
_ids=('Test', upload_response['data']['id']))
url = 'https://waterbutler.io/{}/children/'.format(
path.parent.identifier)
aiohttpretty.register_json_uri('POST',
url,
status=201,
body=upload_response)
inner_provider.metadata = utils.MockCoroutine(
return_value=utils.MockFileMetadata())
inner_provider.upload.side_effect = Exception()
with pytest.raises(Exception):
await provider.upload(file_stream, path)
assert not os.path.isfile(FILE_PATH_PENDING + '/patched_path')
inner_provider.upload.assert_called_once_with(
file_stream,
WaterButlerPath('/patched_path'),
check_created=False,
fetch_metadata=False)
def test_rename(self):
metadata = utils.MockFileMetadata()
self.source_provider.move = utils.MockCoroutine(
return_value=(metadata, False)
)
payload = self.payload()
payload['rename'] = 'MyCoolFileGuys'
resp = yield self.http_client.fetch(
self.get_url('/ops/move'),
method='POST',
body=json.dumps(payload)
)
assert resp.code == 200
assert json.loads(resp.body.decode()) == metadata.serialized()
assert self.source_provider.move.called
self.source_provider.move.assert_called_once_with(
self.destination_provider,
WaterButlerPath(payload['source']['path']),
WaterButlerPath(payload['destination']['path']),
rename='MyCoolFileGuys',
conflict='replace'
)
async def test_upload_existing(self, monkeypatch, provider_and_mock, file_stream, upload_path,
upload_response, mock_time):
self.patch_tasks(monkeypatch)
provider, inner_provider = provider_and_mock
url = 'https://waterbutler.io/{}/children/'.format(upload_path.parent.identifier)
inner_provider.move.return_value = (utils.MockFileMetadata(), True)
inner_provider.metadata.side_effect = exceptions.MetadataError('Boom!', code=404)
aiohttpretty.register_json_uri('POST', url, status=200, body=upload_response)
res, created = await provider.upload(file_stream, upload_path)
assert created is False
assert res.name == '[TEST]'
assert res.extra['version'] == 8
assert res.provider == 'osfstorage'
assert res.extra['downloads'] == 0
assert res.extra['checkout'] is None
assert upload_path.identifier_path == res.path
expected_path = WaterButlerPath('/' + file_stream.writers['sha256'].hexdigest)
inner_provider.metadata.assert_called_once_with(expected_path)
inner_provider.upload.assert_called_once_with(file_stream,
WaterButlerPath('/patched_path'),
check_created=False,
fetch_metadata=False)
inner_provider.move.assert_called_once_with(inner_provider,
WaterButlerPath('/patched_path'),
expected_path)
def test_upload_and_tasks(self, monkeypatch, provider_and_mock, file_stream, credentials, settings, mock_time):
provider, inner_provider = provider_and_mock
basepath = 'waterbutler.providers.osfstorage.provider.{}'
path = WaterButlerPath('/foopath', _ids=('Test', 'OtherTest'))
url = 'https://waterbutler.io/{}/children/'.format(path.parent.identifier)
mock_parity = mock.Mock()
mock_backup = mock.Mock()
inner_provider.move.return_value = (utils.MockFileMetadata(), True)
inner_provider.metadata.side_effect = exceptions.MetadataError('Boom!', code=404)
aiohttpretty.register_json_uri('POST', url, status=201, body={'version': 'versionpk', 'data': {'version': 42, 'downloads': 30, 'path': '/alkjdaslke09', 'checkout': None, 'md5': 'abcd', 'sha256': 'bcde'}})
monkeypatch.setattr(basepath.format('backup.main'), mock_backup)
monkeypatch.setattr(basepath.format('parity.main'), mock_parity)
monkeypatch.setattr(basepath.format('settings.RUN_TASKS'), True)
monkeypatch.setattr(basepath.format('os.rename'), lambda *_: None)
monkeypatch.setattr(basepath.format('uuid.uuid4'), lambda: 'uniquepath')
res, created = yield from provider.upload(file_stream, path)
assert created is True
assert res.name == 'foopath'
assert res.extra['version'] == 42
assert res.provider == 'osfstorage'
assert res.extra['downloads'] == 30
assert res.extra['checkout'] is None
inner_provider.upload.assert_called_once_with(file_stream, WaterButlerPath('/uniquepath'), check_created=False, fetch_metadata=False)
complete_path = os.path.join(FILE_PATH_COMPLETE, file_stream.writers['sha256'].hexdigest)
mock_parity.assert_called_once_with(complete_path, credentials['parity'], settings['parity'])
mock_backup.assert_called_once_with(complete_path, 'versionpk', 'https://waterbutler.io/hooks/metadata/', credentials['archive'], settings['parity'])
inner_provider.metadata.assert_called_once_with(WaterButlerPath('/' + file_stream.writers['sha256'].hexdigest))
inner_provider.move.assert_called_once_with(inner_provider, WaterButlerPath('/uniquepath'), WaterButlerPath('/' + file_stream.writers['sha256'].hexdigest))
def test_upload_existing(self, monkeypatch, provider_and_mock, file_stream, mock_time):
self.patch_tasks(monkeypatch)
provider, inner_provider = provider_and_mock
path = WaterButlerPath('/foopath', _ids=('Test', 'OtherTest'))
url = 'https://waterbutler.io/{}/children/'.format(path.parent.identifier)
inner_provider.move.return_value = (utils.MockFileMetadata(), True)
inner_provider.metadata.side_effect = exceptions.MetadataError('Boom!', code=404)
aiohttpretty.register_json_uri('POST', url, status=200, body={'data': {'downloads': 10, 'version': 8, 'path': '/24601', 'checkout': 'hmoco', 'md5': '1234', 'sha256': '2345'}})
res, created = yield from provider.upload(file_stream, path)
assert created is False
assert res.name == 'foopath'
assert res.path == '/24601'
assert res.extra['version'] == 8
assert res.provider == 'osfstorage'
assert res.extra['downloads'] == 10
assert res.extra['checkout'] == 'hmoco'
inner_provider.metadata.assert_called_once_with(WaterButlerPath('/' + file_stream.writers['sha256'].hexdigest))
inner_provider.upload.assert_called_once_with(file_stream, WaterButlerPath('/uniquepath'), check_created=False, fetch_metadata=False)
inner_provider.move.assert_called_once_with(inner_provider, WaterButlerPath('/uniquepath'), WaterButlerPath('/' + file_stream.writers['sha256'].hexdigest))
async def test_upload_new(self, monkeypatch, provider_and_mock,
file_stream, upload_response, mock_time):
self.patch_tasks(monkeypatch)
path = WaterButlerPath('/newfile', _ids=('rootId', None))
url = 'https://waterbutler.io/{}/children/'.format(
path.parent.identifier)
aiohttpretty.register_json_uri('POST',
url,
status=201,
body=upload_response)
provider, inner_provider = provider_and_mock
inner_provider.metadata = utils.MockCoroutine(
return_value=utils.MockFileMetadata())
res, created = await provider.upload(file_stream, path)
assert created is True
assert res.name == 'newfile'
assert res.extra['version'] == 8
assert res.provider == 'osfstorage'
assert res.extra['downloads'] == 10
assert res.extra['checkout'] is None
assert path.identifier_path == res.path
inner_provider.delete.assert_called_once_with(
WaterButlerPath('/uniquepath'))
inner_provider.metadata.assert_called_once_with(
WaterButlerPath('/' + file_stream.writers['sha256'].hexdigest))
inner_provider.upload.assert_called_once_with(
file_stream,
WaterButlerPath('/uniquepath'),
check_created=False,
fetch_metadata=False)
def test_get_not_coro(self):
expected = [utils.MockFileMetadata(), utils.MockFolderMetadata()]
self.mock_provider.revisions = mock.Mock(return_value=expected)
resp = yield self.http_client.fetch(
self.get_url('/revisions?provider=queenhub&path=/brian.tiff'), )
assert {
'data': [m.serialized() for m in expected]
} == json.loads(resp.body.decode())
async def test_upload_and_tasks(self, monkeypatch, provider_and_mock,
file_stream, upload_response, credentials,
settings, mock_time):
provider, inner_provider = provider_and_mock
basepath = 'waterbutler.providers.osfstorage.provider.{}'
path = WaterButlerPath('/' + upload_response['data']['name'],
_ids=('Test', upload_response['data']['id']))
url = 'https://waterbutler.io/{}/children/'.format(
path.parent.identifier)
# mock_parity = mock.Mock()
# mock_backup = mock.Mock()
inner_provider.move.return_value = (utils.MockFileMetadata(), True)
inner_provider.metadata.side_effect = exceptions.MetadataError(
'Boom!', code=404)
aiohttpretty.register_json_uri('POST',
url,
status=201,
body=upload_response)
# monkeypatch.setattr(basepath.format('backup._push_file_archive'), mock_backup)
# monkeypatch.setattr(basepath.format('parity._parity_create_files'), mock_parity)
monkeypatch.setattr(basepath.format('settings.RUN_TASKS'), False)
monkeypatch.setattr(basepath.format('os.rename'), lambda *_: None)
monkeypatch.setattr(basepath.format('uuid.uuid4'),
lambda: 'uniquepath')
res, created = await provider.upload(file_stream, path)
assert created is True
assert res.name == '[TEST]'
assert res.extra['version'] == 8
assert res.provider == 'osfstorage'
assert res.extra['downloads'] == 0
assert res.extra['checkout'] is None
inner_provider.upload.assert_called_once_with(
file_stream,
WaterButlerPath('/uniquepath'),
check_created=False,
fetch_metadata=False)
# complete_path = os.path.join(FILE_PATH_COMPLETE, local_complete_dir,
# file_stream.writers['sha256'].hexdigest)
# mock_parity.assert_called_once_with(complete_path, upload_response['version'],
# 'https://waterbutler.io/hooks/metadata/',
# credentials['parity'], settings['parity'])
# mock_backup.assert_called_once_with(complete_path, upload_response['version'],
# 'https://waterbutler.io/hooks/metadata/',
# credentials['archive'], settings['archive'])
expected_path = WaterButlerPath(
'/' + file_stream.writers['sha256'].hexdigest)
inner_provider.metadata.assert_called_once_with(expected_path)
inner_provider.move.assert_called_once_with(
inner_provider, WaterButlerPath('/uniquepath'), expected_path)
def test_return_values(self, event_loop, providers, bundles, callback, src_path, dest_path, mock_time):
src, dest = providers
src_bundle, dest_bundle = bundles
metadata = test_utils.MockFileMetadata()
src.copy.return_value = (metadata, False)
ret1, ret2 = copy.copy(cp.deepcopy(src_bundle), cp.deepcopy(dest_bundle))
assert (ret1, ret2) == (metadata, False)
(method, url, data), _ = callback.call_args_list[0]
assert method == 'PUT'
assert url == 'dest_callback'
assert data['action'] == 'copy'
assert data['auth'] == {'callback_url': 'dest_callback'}
assert data['email'] == False
assert data['errors'] == []
assert data['time'] == FAKE_TIME + 60
assert data['source'] == {
'nid': 'mst3k',
'resource': 'mst3k',
'path': '/' + src_path.raw_path,
'name': src_path.name,
'materialized': str(src_path),
'provider': src.NAME,
'kind': 'file',
'extra': {},
}
assert data['destination'] == {
'nid': 'fbi4u',
'resource': 'fbi4u',
'path': metadata.path,
'name': metadata.name,
'materialized': metadata.path,
'provider': dest.NAME,
'kind': 'file',
'contentType': metadata.content_type,
'etag': hashlib.sha256(
'{}::{}'.format(metadata.provider, metadata.etag)
.encode('utf-8')
).hexdigest(),
'extra': metadata.extra,
'modified': metadata.modified,
'modified_utc': metadata.modified_utc,
'created_utc': metadata.created_utc,
'size': metadata.size,
}
def test_calls_move(self):
self.source_provider.move = utils.MockCoroutine(
return_value=(utils.MockFileMetadata(), False))
yield self.http_client.fetch(self.get_url('/ops/move'),
method='POST',
body=json.dumps(self.payload()))
assert self.source_provider.move.called
self.source_provider.move.assert_called_once_with(
self.destination_provider,
WaterButlerPath(self.payload()['source']['path']),
WaterButlerPath(self.payload()['destination']['path']),
rename=None,
conflict='replace')
def test_conflict(self):
self.source_provider.move = utils.MockCoroutine(
return_value=(utils.MockFileMetadata(), True))
payload = self.payload()
payload['conflict'] = 'keep'
resp = yield self.http_client.fetch(self.get_url('/ops/move'),
method='POST',
body=json.dumps(payload))
assert resp.code == 201
assert self.source_provider.move.called
self.source_provider.move.assert_called_once_with(
self.destination_provider,
WaterButlerPath(payload['source']['path']),
WaterButlerPath(payload['destination']['path']),
rename=None,
conflict='keep')
def test_upload(self):
data = b'stone cold crazy'
expected = utils.MockFileMetadata()
self.mock_provider.upload = utils.MockCoroutine(return_value=(expected, True))
resp = yield self.http_client.fetch(
self.get_url('/file?provider=queenhub&path=/roger.png'),
method='PUT',
body=data,
)
calls = self.mock_provider.upload.call_args_list
assert len(calls) == 1
args, kwargs = calls[0]
assert isinstance(args[0], streams.RequestStreamReader)
streamed = asyncio.new_event_loop().run_until_complete(args[0].read())
assert streamed == data
assert kwargs['action'] == 'upload'
assert str(kwargs['path']) == '/roger.png'
assert expected.serialized() == json.loads(resp.body.decode())
def test_folder_json_api_serialize(self):
folder_metadata = utils.MockFolderMetadata()
folder_metadata.children = [utils.MockFileMetadata()]
serialized = folder_metadata.json_api_serialized('n0d3z')
child = serialized['attributes']['children'][0]
etag = hashlib.sha256('{}::{}'.format(
'mock', 'etag').encode('utf-8')).hexdigest()
assert len(serialized['attributes']['children']) == 1
assert child == {
'extra': {},
'kind': 'file',
'name': 'Foo.name',
'path': '/Foo.name',
'provider': 'mock',
'materialized': '/Foo.name',
'etag': etag,
'contentType': 'application/octet-stream',
'modified': 'never',
'size': 1337,
}
def dest_provider():
p = test_utils.MockProvider()
p.copy.return_value = (test_utils.MockFileMetadata(), True)
p.auth['callback_url'] = 'dest_callback'
return p
def src_provider():
p = test_utils.MockProvider()
p.move.return_value = (test_utils.MockFileMetadata(), True)
return p
def test_file_metadata(self):
file_metadata = utils.MockFileMetadata()
assert file_metadata.is_file == True
assert file_metadata.is_folder == False
def dest_provider():
p = test_utils.MockProvider()
p.copy.return_value = (test_utils.MockFileMetadata(), True)
return p
def src_provider():
p = test_utils.MockProvider()
p.move.return_value = (test_utils.MockFileMetadata(), True)
p.auth['callback_url'] = 'src_callback'
return p