def setUp(self):
super().setUp()
self.task_id = str(uuid.uuid4())
self.handle_retries = Mock()
self.dir_manager = DirManager(self.tempdir)
self.resource_manager = HyperdriveResourceManager(self.dir_manager)
self.resource_manager._handle_retries = self.handle_retries
file_name = 'test_file'
file_path = os.path.join(self.tempdir, file_name)
Path(file_path).touch()
self.files = {file_path: file_name}
def test_download_options(self, *_):
dm = DirManager(self.path)
rm = HyperdriveResourceManager(dm)
self.client.resource_server.resource_manager = rm
ts = self.ts
options = HyperdriveClientOptions(HyperdriveClient.CLIENT_ID,
HyperdriveClient.VERSION)
client_options = ts.get_download_options(options, task_id='task_id')
assert client_options.peers is None
peers = [
to_hyperg_peer('127.0.0.1', 3282),
to_hyperg_peer('127.0.0.1', 0),
to_hyperg_peer('127.0.0.1', None),
to_hyperg_peer('1.2.3.4', 3282), {
'uTP': ('1.2.3.4', 3282)
}
]
options = HyperdriveClientOptions(HyperdriveClient.CLIENT_ID,
HyperdriveClient.VERSION,
options=dict(peers=peers))
client_options = ts.get_download_options(options, task_id='task_id')
assert client_options.options.get('peers') == [
to_hyperg_peer('127.0.0.1', 3282),
to_hyperg_peer('1.2.3.4', 3282),
]
def __init__(self,
node_name,
node,
keys_auth,
listen_address="",
listen_port=0,
root_path="res",
use_distributed_resources=True,
tasks_dir="tasks",
task_persistence=False):
super(TaskManager, self).__init__()
self.apps_manager = AppsManager()
self.apps_manager.load_apps()
apps = self.apps_manager.apps.values()
task_types = [app.task_type_info(None, app.controller) for app in apps]
self.task_types = {t.name.lower(): t for t in task_types}
self.node_name = node_name
self.node = node
self.keys_auth = keys_auth
self.key_id = keys_auth.get_key_id()
self.tasks = {}
self.tasks_states = {}
self.subtask2task_mapping = {}
self.listen_address = listen_address
self.listen_port = listen_port
# FIXME Remove this variable and make task persistance obligatory after it is more tested
# Remember to also remove it from init params
self.task_persistence = task_persistence
self.tasks_dir = Path(tasks_dir)
if not self.tasks_dir.is_dir():
self.tasks_dir.mkdir(parents=True)
self.root_path = root_path
self.dir_manager = DirManager(self.get_task_manager_root())
# resource_manager = OpenStackSwiftResourceManager(self.dir_manager,
# resource_dir_method=self.dir_manager.get_task_temporary_dir)
resource_manager = HyperdriveResourceManager(
self.dir_manager,
resource_dir_method=self.dir_manager.get_task_temporary_dir)
self.task_result_manager = EncryptedResultPackageManager(
resource_manager)
self.activeStatus = [
TaskStatus.computing, TaskStatus.starting, TaskStatus.waiting
]
self.use_distributed_resources = use_distributed_resources
self.comp_task_keeper = CompTaskKeeper(self.tasks_dir,
persist=self.task_persistence)
if self.task_persistence:
self.restore_tasks()
def __init__(self,
node_name,
node,
keys_auth,
root_path,
use_distributed_resources=True,
tasks_dir="tasks",
task_persistence=True,
apps_manager=AppsManager(),
finished_cb=None):
super().__init__()
self.apps_manager = apps_manager
apps = list(apps_manager.apps.values())
task_types = [app.task_type_info() for app in apps]
self.task_types = {t.name.lower(): t for t in task_types}
self.node_name = node_name
self.node = node
self.keys_auth = keys_auth
self.key_id = keys_auth.key_id
self.tasks: Dict[str, Task] = {}
self.tasks_states: Dict[str, TaskState] = {}
self.subtask2task_mapping: Dict[str, str] = {}
self.task_persistence = task_persistence
tasks_dir = Path(tasks_dir)
self.tasks_dir = tasks_dir / "tmanager"
if not self.tasks_dir.is_dir():
self.tasks_dir.mkdir(parents=True)
self.root_path = root_path
self.dir_manager = DirManager(self.get_task_manager_root())
resource_manager = HyperdriveResourceManager(
self.dir_manager,
resource_dir_method=self.dir_manager.get_task_temporary_dir,
)
self.task_result_manager = EncryptedResultPackageManager(
resource_manager)
self.activeStatus = [
TaskStatus.computing, TaskStatus.starting, TaskStatus.waiting
]
self.use_distributed_resources = use_distributed_resources
self.comp_task_keeper = CompTaskKeeper(
tasks_dir,
persist=self.task_persistence,
)
self.requestor_stats_manager = RequestorTaskStatsManager()
self.finished_cb = finished_cb
if self.task_persistence:
self.restore_tasks()
def __create_task_server(session):
from golem.clientconfigdescriptor import ClientConfigDescriptor
from golem.task.taskserver import TaskServer
client = Mock(datadir=session.data_dir)
dir_manager = DirManager(session.data_dir)
resource_manager = HyperdriveResourceManager(dir_manager=dir_manager)
resource_manager.successful_uploads = True
resource_manager.successful_downloads = True
resource_manager.add_file_org = resource_manager.add_file
resource_manager.add_file = types.MethodType(_add_file,
resource_manager)
resource_manager.pull_resource_org = resource_manager.pull_resource
resource_manager.pull_resource = types.MethodType(_pull_resource,
resource_manager)
with patch("golem.network.concent.handlers_library"
".HandlersLibrary"
".register_handler"):
task_server = TaskServer(
node=Mock(client=client, key=str(uuid.uuid4())),
config_desc=ClientConfigDescriptor(),
client=client,
use_docker_manager=False
)
task_server.task_manager = Mock(
task_result_manager=Mock(
resource_manager=resource_manager
)
)
peers = to_hyperg_peer('127.0.0.1', 3282)
client_options = HyperdriveClientOptions(
HyperdriveClient.CLIENT_ID,
HyperdriveClient.VERSION,
options=dict(
peers=peers,
filtered=peers
)
)
task_server.get_share_options = Mock(return_value=client_options)
task_server.get_download_options = Mock(return_value=client_options)
session.task_server = task_server
class TestHyperdriveResourceManager(TempDirFixture):
def setUp(self):
super().setUp()
self.task_id = str(uuid.uuid4())
self.handle_retries = Mock()
self.dir_manager = DirManager(self.tempdir)
self.resource_manager = HyperdriveResourceManager(self.dir_manager)
self.resource_manager._handle_retries = self.handle_retries
file_name = 'test_file'
file_path = os.path.join(self.tempdir, file_name)
Path(file_path).touch()
self.files = {file_path: file_name}
def test_add_files_invalid_paths(self, add, restore):
files = {str(uuid.uuid4()): 'does_not_exist'}
with self.assertRaises(ResourceError):
self.resource_manager.add_files(files,
self.task_id,
resource_hash=None)
assert not add.called
assert not restore.called
def test_add_files_empty_resource_hash(self, add, restore):
self.resource_manager.add_files(self.files,
self.task_id,
resource_hash=None)
assert not restore.called
assert add.called
def test_add_files_with_resource_hash(self, add, restore):
self.resource_manager.add_files(self.files,
self.task_id,
resource_hash=str(uuid.uuid4()))
assert restore.called
assert not add.called
def test_add_task_failure(self, _add, _restore):
exc = Exception('Test exception')
self.resource_manager._add_files = Mock(side_effect=exc)
deferred = self.resource_manager.add_task(self.files, self.task_id)
assert deferred.called
assert isinstance(deferred.result, Failure)
def start_network(self):
log.info("Starting network ...")
self.node.collect_network_info(self.config_desc.seed_host,
use_ipv6=self.config_desc.use_ipv6)
log.debug("Is super node? %s", self.node.is_super_node())
# self.ipfs_manager = IPFSDaemonManager(
# connect_to_bootstrap_nodes=self.connect_to_known_hosts)
# self.ipfs_manager.store_client_info()
self.p2pservice = P2PService(
self.node,
self.config_desc,
self.keys_auth,
connect_to_known_hosts=self.connect_to_known_hosts)
self.task_server = TaskServer(
self.node,
self.config_desc,
self.keys_auth,
self,
use_ipv6=self.config_desc.use_ipv6,
use_docker_machine_manager=self.use_docker_machine_manager)
dir_manager = self.task_server.task_computer.dir_manager
log.info("Starting resource server ...")
self.daemon_manager = HyperdriveDaemonManager(self.datadir)
hyperdrive_ports = self.daemon_manager.start()
resource_manager = HyperdriveResourceManager(dir_manager)
self.resource_server = BaseResourceServer(resource_manager,
dir_manager, self.keys_auth,
self)
def connect((p2p_port, task_port)):
log.info('P2P server is listening on port %s', p2p_port)
log.info('Task server is listening on port %s', task_port)
dispatcher.send(signal='golem.p2p',
event='listening',
port=[p2p_port, task_port] +
list(hyperdrive_ports))
listener = ClientTaskComputerEventListener(self)
self.task_server.task_computer.register_listener(listener)
self.p2pservice.connect_to_network()
if self.monitor:
self.diag_service.register(self.p2pservice,
self.monitor.on_peer_snapshot)
self.monitor.on_login()
def terminate(*exceptions):
log.error("Golem cannot listen on ports: %s", exceptions)
self.quit()
task = Deferred()
p2p = Deferred()
gatherResults([p2p, task],
consumeErrors=True).addCallbacks(connect, terminate)
log.info("Starting p2p server ...")
self.p2pservice.task_server = self.task_server
self.p2pservice.set_resource_server(self.resource_server)
self.p2pservice.set_metadata_manager(self)
self.p2pservice.start_accepting(listening_established=p2p.callback,
listening_failure=p2p.errback)
log.info("Starting task server ...")
self.task_server.start_accepting(listening_established=task.callback,
listening_failure=task.errback)