def test_get_attached_files__files_exist(database_instance, mocker): # pylint: disable=redefined-outer-name
class MockAttachedFilesListCache:
namespace = 'namespace'
did = 'did'
expiry = 0
@staticmethod
def get_or_none(*args, **kwargs): # pylint: disable=unused-argument
file_dids = [{
'did': 'did1',
'size': 1
}, {
'did': 'did2',
'size': 3
}]
file_dids_json = json.dumps(file_dids)
return Struct(file_dids=file_dids_json)
mocker.patch('rucio_jupyterlab.db.AttachedFilesListCache',
MockAttachedFilesListCache)
result = database_instance.get_attached_files('namespace', 'did')
result_dict = [x.__dict__ for x in result]
expected_result = [
AttachedFile(did='did1', size=1),
AttachedFile(did='did2', size=3)
]
expected_result_dict = [x.__dict__ for x in expected_result]
assert result_dict == expected_result_dict, "Invalid return value"
for res in result:
assert isinstance(
res, AttachedFile), "Return array element is not AttachedFile"
def test_get_files__cache_exist__no_force_fetch(mocker, rucio):
"""
If cache exist and force_fetch is false, assert method
to call db.get_attached_files and NOT rucio.get_replicas.
Assert their call parameters as well.
"""
mock_db = MockDatabaseInstance()
mocker.patch('rucio_jupyterlab.handlers.did_browser.get_db',
return_value=mock_db)
mocker.patch.object(rucio, 'get_replicas', return_value=[])
mock_attached_files = [
AttachedFile('scope1:name1', 123456),
AttachedFile('scope2:name2', 789456),
AttachedFile('scope3:name3', 1)
]
mocker.patch.object(mock_db,
'get_attached_files',
return_value=mock_attached_files)
handler = DIDBrowserHandlerImpl(MOCK_ACTIVE_INSTANCE, rucio)
result = handler.get_files('scope', 'name', False)
rucio.get_replicas.assert_not_called()
mock_db.get_attached_files.assert_called_once_with(
namespace=MOCK_ACTIVE_INSTANCE, did='scope:name') # pylint: disable=no-member
expected = [x.__dict__ for x in mock_attached_files]
assert result == expected, "Invalid return value"
def test_set_attached_files(database_instance, mocker): # pylint: disable=redefined-outer-name
class MockAttachedFilesListCache:
@staticmethod
def execute(*args, **kwargs): # pylint: disable=unused-argument
pass
@staticmethod
def replace(namespace, did, file_dids, expiry):
assert namespace == 'namespace', "Invalid namespace"
assert did == 'scope:name', "Invalid DID"
expected_file_dids = [{
'did': 'did1',
'size': 1
}, {
'did': 'did2',
'size': 3
}]
assert json.loads(
file_dids) == expected_file_dids, "Invalid file DIDs"
assert expiry > int(
time.time()), "Expiry should be later than current time"
return MockAttachedFilesListCache
mocker.patch('rucio_jupyterlab.db.AttachedFilesListCache',
MockAttachedFilesListCache)
attached_files = [
AttachedFile(did='did1', size=1),
AttachedFile(did='did2', size=3)
]
database_instance.set_attached_files('namespace', 'scope:name',
attached_files)
def test_get_handler(mocker, rucio):
mock_self = MockHandler()
def mock_get_query_argument(key, default=None):
args = {
'namespace': MOCK_ACTIVE_INSTANCE,
'poll': '0',
'did': 'scope:name'
}
return args.get(key, default)
mocker.patch.object(mock_self,
'get_query_argument',
side_effect=mock_get_query_argument)
rucio_api_factory = RucioAPIFactory(None)
mocker.patch.object(rucio_api_factory, 'for_instance', return_value=rucio)
mock_self.rucio = rucio_api_factory
mock_attached_files = [
AttachedFile('scope1:name1', 123456),
AttachedFile('scope2:name2', 789456),
AttachedFile('scope3:name3', 1)
]
class MockDIDBrowserHandlerImpl(DIDBrowserHandlerImpl):
def get_files(self, scope, name, force_fetch=False):
return [x.__dict__ for x in mock_attached_files]
mocker.patch('rucio_jupyterlab.handlers.did_browser.DIDBrowserHandlerImpl',
MockDIDBrowserHandlerImpl)
def finish_side_effect(output):
finish_json = json.loads(output)
assert finish_json == [x.__dict__ for x in mock_attached_files
], "Invalid finish response"
mocker.patch.object(mock_self, 'finish', side_effect=finish_side_effect)
DIDBrowserHandler.get(mock_self)
calls = [call('namespace'), call('poll', '0'), call('did')]
mock_self.get_query_argument.assert_has_calls(calls, any_order=True) # pylint: disable=no-member
rucio_api_factory.for_instance.assert_called_once_with(
MOCK_ACTIVE_INSTANCE) # pylint: disable=no-member
def get_files(self, scope, name, force_fetch=False):
parent_did = f'{scope}:{name}'
attached_files = self.db.get_attached_files(
namespace=self.namespace,
did=parent_did) if not force_fetch else None
if attached_files:
return [d.__dict__ for d in attached_files]
file_dids = self.rucio.get_replicas(scope, name)
attached_files = [
AttachedFile(did=(d.get('scope') + ':' + d.get('name')),
size=d.get('bytes')) for d in file_dids
]
self.db.set_attached_files(self.namespace, parent_did, attached_files)
return [d.__dict__ for d in attached_files]
def fetch_attached_pfn_file_replicas(self, scope, name):
pfn_file_replicas = []
attached_dids = []
fetched_file_replicas = self.fetch_file_replicas(scope, name)
for pfn_file_replica in fetched_file_replicas:
self.db.set_file_replica(self.namespace,
file_did=pfn_file_replica.did,
pfn=pfn_file_replica.pfn,
size=pfn_file_replica.size)
pfn_file_replicas.append(pfn_file_replica)
attached_dids.append(
AttachedFile(did=pfn_file_replica.did,
size=pfn_file_replica.size))
did = scope + ':' + name
self.db.set_attached_files(self.namespace, did, attached_dids)
return pfn_file_replicas
def test_get_files__cache_exist__force_fetch(mocker, rucio):
"""
If cache exist and force_fetch is true, assert method
to NOT call db.get_attached_files and call rucio.get_replicas.
Assert their call parameters as well.
"""
mock_db = MockDatabaseInstance()
mocker.patch('rucio_jupyterlab.handlers.did_browser.get_db',
return_value=mock_db)
mock_replicas = [{
'scope': 'scope1',
'name': 'name1',
'bytes': 123456
}, {
'scope': 'scope2',
'name': 'name2',
'bytes': 789456
}, {
'scope': 'scope3',
'name': 'name3',
'bytes': 1
}]
mock_attached_files = [
AttachedFile(did=(d.get('scope') + ':' + d.get('name')),
size=d.get('bytes')) for d in mock_replicas
]
mocker.patch.object(rucio, 'get_replicas', return_value=mock_replicas)
mocker.patch.object(mock_db,
'get_attached_files',
return_value=mock_attached_files)
handler = DIDBrowserHandlerImpl(MOCK_ACTIVE_INSTANCE, rucio)
result = handler.get_files('scope', 'name', True)
rucio.get_replicas.assert_called_once_with('scope', 'name')
mock_db.get_attached_files.assert_not_called() # pylint: disable=no-member
expected = [x.__dict__ for x in mock_attached_files]
assert result == expected, "Invalid return value"
def mapper(d, _):
return AttachedFile(did=(d.get('scope') + ':' + d.get('name')),
size=d.get('bytes'))
def get_attached_files(self, namespace, did):
return [
AttachedFile(did='scope:name1', size=123456),
AttachedFile(did='scope:name2', size=123456),
AttachedFile(did='scope:name3', size=123456)
]
# Copyright European Organization for Nuclear Research (CERN)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Muhammad Aditya Hilmy, <*****@*****.**>, 2020
from rucio_jupyterlab.entity import AttachedFile
from rucio_jupyterlab.mode_handlers.replica import ReplicaModeHandler
from .mocks.mock_db import MockDatabaseInstance, Struct
MOCK_ATTACHED_FILES = [
AttachedFile(did='scope:name1', size=123),
AttachedFile(did='scope:name2', size=456),
AttachedFile(did='scope:name3', size=789)
]
mock_rucio_replicas_all_available = [
{
"adler32": "01a100a2",
"name": "name1",
"rses": {
"SWAN-EOS": [
"root://xrd1:1094//eos/docker/user/rucio/scope:name1"
]
},
"bytes": 123,
"states": {