def test_restart_task(self, mock_addr):
mock_addr.return_value = self.addr_return
with self.assertLogs(logger, level="WARNING"):
assert self.tm.restart_task("xyz") is None
t = self._get_task_mock()
sync_wait(self.tm.add_new_task(t))
with self.assertNoLogs(logger, level="WARNING"):
self.tm.restart_task("xyz")
assert self.tm.tasks["xyz"].task_status == TaskStatus.waiting
assert self.tm.tasks_states["xyz"].status == TaskStatus.waiting
with patch('golem.task.taskbase.Task.needs_computation',
return_value=True):
self.tm.get_next_subtask("NODEID", "NODENAME", "xyz", 1000, 100,
10000, 10000)
t.query_extra_data_return_value.ctd.subtask_id = "xxyyzz2"
self.tm.get_next_subtask("NODEID2", "NODENAME2", "xyz", 1000, 100,
10000, 10000)
self.assertEquals(len(self.tm.tasks_states["xyz"].subtask_states),
2)
with self.assertNoLogs(logger, level="WARNING"):
self.tm.restart_task("xyz")
assert self.tm.tasks["xyz"].task_status == TaskStatus.waiting
assert self.tm.tasks_states["xyz"].status == TaskStatus.waiting
assert len(self.tm.tasks_states["xyz"].subtask_states) == 2
for ss in self.tm.tasks_states["xyz"].subtask_states.values():
assert ss.subtask_status == SubtaskStatus.restarted
def test_general_exception(self, start_mock, log_mock, *_):
start_mock.side_effect = RuntimeError("TEST ERROR")
task = self.client.task_manager.create_task(self.t_dict)
deferred = rpc.enqueue_new_task(self.client, task)
with self.assertRaises(RuntimeError):
golem_deferred.sync_wait(deferred)
log_mock.assert_called_once()
def _restart_subtasks(
client,
old_task_id,
task_dict,
subtask_ids_to_copy,
force,
):
@defer.inlineCallbacks
@safe_run(
lambda e: logger.error(
'Restarting subtasks_failed. task_dict=%r, subtask_ids_to_copy=%r',
task_dict,
subtask_ids_to_copy,
), )
def deferred():
new_task = yield enqueue_new_task(
client=client,
task=client.task_manager.create_task(task_dict),
force=force,
)
client.task_manager.copy_results(
old_task_id=old_task_id,
new_task_id=new_task.header.task_id,
subtask_ids_to_copy=subtask_ids_to_copy)
# Function passed to twisted.threads.deferToThread can't itself
# return a deferred, that's why I defined inner deferred function
# and use sync_wait below.
golem_deferred.sync_wait(deferred())
def test_get_balance(self, async_run, *_):
c = self.client
result = (None, None, None)
deferred = Deferred()
deferred.result = result
deferred.called = True
async_run.return_value = deferred
c.transaction_system = Mock()
c.transaction_system.get_balance.return_value = result
balance = sync_wait(c.get_balance())
assert balance == (None, None, None)
result = (None, 1, None)
deferred.result = result
balance = sync_wait(c.get_balance())
assert balance == (None, None, None)
result = (1, 1, None)
deferred.result = result
balance = sync_wait(c.get_balance())
assert balance == (u"1", u"1", u"None")
assert all(isinstance(entry, unicode) for entry in balance)
c.transaction_system = None
balance = sync_wait(c.get_balance())
assert balance == (None, None, None)
def account():
client = account.client
node = sync_wait(account.client.get_node())
node_key = node['key']
computing_trust = sync_wait(client.get_computing_trust(node_key))
requesting_trust = sync_wait(client.get_requesting_trust(node_key))
payment_address = sync_wait(client.get_payment_address())
balance = sync_wait(client.get_balance())
if any(b is None for b in balance):
balance = 0, 0, 0
gnt_balance, gnt_available, eth_balance = balance
gnt_balance = float(gnt_balance)
gnt_available = float(gnt_available)
eth_balance = float(eth_balance)
gnt_reserved = gnt_balance - gnt_available
return dict(node_name=node['node_name'],
Golem_ID=node_key,
requestor_reputation=int(requesting_trust * 100),
provider_reputation=int(computing_trust * 100),
finances=dict(eth_address=payment_address,
total_balance=_fmt(gnt_balance),
available_balance=_fmt(gnt_available),
reserved_balance=_fmt(gnt_reserved),
eth_balance=_fmt(eth_balance, unit="ETH")))
def test_subsampled_image(self):
d = Deferred()
def success(*args, **kwargs):
# pylint: disable=unused-argument
assert False
def failure(*args, **kwargs):
# pylint: disable=unused-argument
d.callback(True)
verification_data = {}
verification_data['subtask_info'] = self.subtask_info
verification_data['results'] = []
verification_data['reference_data'] = []
verification_data['resources'] = self.resources
verification_data['paths'] = os.path.dirname(self.resources[0])
verifier = BlenderRenderTask.VERIFIER_CLASS(verification_data)
verifier.default_crops_number = 1
verifier.current_results_files = \
['tests/apps/blender/verification/test_data/almost_good_image.png']
verifier.subtask_info = self.subtask_info
verifier.resources = self.resources
finished = self.blender_reference_generator.render_crops(
self.resources, self.subtask_info, 1)
for deferred in finished:
deferred.addCallback(success)
deferred.addErrback(failure)
sync_wait(d, TestVerificatorModuleIntegration.TIMEOUT)
def test_results(self, trust, mock_addr, dump_mock):
mock_addr.return_value = ("10.10.10.10", 1111, "Full NAT")
ccd = self._get_config_desc()
ts = TaskServer(Node(), ccd, EllipticalKeysAuth(self.path), self.client,
use_docker_machine_manager=False)
self.ts = ts
ts.task_manager.listen_port = 1111
ts.task_manager.listen_address = "10.10.10.10"
ts.receive_subtask_computation_time("xxyyzz", 1031)
extra_data = Mock()
extra_data.ctd = ComputeTaskDef()
extra_data.ctd.task_id = "xyz"
extra_data.ctd.subtask_id = "xxyyzz"
extra_data.ctd.environment = "DEFAULT"
extra_data.should_wait = False
task_mock = get_mock_task("xyz", "xxyyzz")
task_mock.query_extra_data.return_value = extra_data
sync_wait(ts.task_manager.add_new_task(task_mock))
ts.task_manager.tasks_states["xyz"].status = ts.task_manager.activeStatus[0]
subtask, wrong_task, wait = ts.task_manager.get_next_subtask("DEF", "DEF", "xyz",
1000, 10, 5, 10, 2,
"10.10.10.10")
ts.receive_subtask_computation_time("xxyyzz", 1031)
self.assertEqual(ts.task_manager.tasks_states["xyz"].subtask_states["xxyyzz"].computation_time, 1031)
expected_value = ceil(1031 * 10 / 3600)
self.assertEqual(ts.task_manager.tasks_states["xyz"].subtask_states["xxyyzz"].value, expected_value)
account_info = Mock()
account_info.key_id = "key"
prev_calls = trust.COMPUTED.increase.call_count
ts.accept_result("xxyyzz", account_info)
ts.client.transaction_system.add_payment_info.assert_called_with("xyz", "xxyyzz", expected_value, account_info)
self.assertGreater(trust.COMPUTED.increase.call_count, prev_calls)
def test_results_no_payment_addr(self, trust, mock_addr, dump_mock):
mock_addr.return_value = ("10.10.10.10", 1111, "Full NAT")
# FIXME: This test is too heavy, it starts up whole Golem Client.
ccd = self._get_config_desc()
ts = TaskServer(Node(), ccd, EllipticalKeysAuth(self.path), self.client,
use_docker_machine_manager=False)
ts.task_manager.listen_address = "10.10.10.10"
ts.task_manager.listen_port = 1111
ts.receive_subtask_computation_time("xxyyzz", 1031)
self.ts = ts
extra_data = Mock()
extra_data.ctd = ComputeTaskDef()
extra_data.ctd.task_id = "xyz"
extra_data.ctd.subtask_id = "xxyyzz"
extra_data.ctd.environment = "DEFAULT"
extra_data.should_wait = False
task_mock = get_mock_task("xyz", "xxyyzz")
task_mock.query_extra_data.return_value = extra_data
sync_wait(ts.task_manager.add_new_task(task_mock))
ts.task_manager.tasks_states["xyz"].status = ts.task_manager.activeStatus[0]
subtask, wrong_task, wait = ts.task_manager.get_next_subtask(
"DEF", "DEF", "xyz", 1000, 10, 5, 10, 2, "10.10.10.10")
ts.receive_subtask_computation_time("xxyyzz", 1031)
account_info = Mock()
account_info.key_id = "key"
account_info.eth_account = Mock()
account_info.eth_account.address = None
ts.accept_result("xxyyzz", account_info)
self.assertEqual(ts.client.transaction_system.add_payment_info.call_count, 0)
def test_ethereum_error(self, deposit_mock, log_mock, *_):
from golem.ethereum import exceptions as eth_exceptions
deposit_mock.side_effect = eth_exceptions.EthereumError('TEST ERROR')
task = self.client.task_manager.create_task(self.t_dict)
deferred = rpc.enqueue_new_task(self.client, task)
with self.assertRaises(eth_exceptions.EthereumError):
golem_deferred.sync_wait(deferred)
log_mock.assert_called_once()
def test_dummy_real_task(self, mock_dtt):
mock_dtt.return_value = 1.0
task = self._get_test_task()
ctd = task.query_extra_data(1.0).ctd
print(ctd)
print(type(ctd))
from twisted.internet import reactor
d = Deferred()
computer = LocalComputer(
root_path=self.tempdir,
success_callback=Mock(),
error_callback=Mock(),
compute_task_def=ctd,
resources=task.task_resources,
)
computer.run()
computer.tt.join()
output = self._extract_results(computer, ctd['subtask_id'])
task.create_reference_data_for_task_validation()
def success(*args, **kwargs):
# pylint: disable=unused-argument
is_subtask_verified = task.verify_subtask(ctd['subtask_id'])
self.assertTrue(is_subtask_verified)
self.assertEqual(task.num_tasks_received, 1)
d.callback(True)
# assert good results - should pass
self.assertEqual(task.num_tasks_received, 0)
task.computation_finished(ctd['subtask_id'], [str(output)],
verification_finished=success)
reactor.iterate()
sync_wait(d, 40)
b = Deferred()
def failure(*args, **kwargs):
# pylint: disable=unused-argument
self.assertFalse(task.verify_subtask(ctd['subtask_id']))
self.assertEqual(task.num_tasks_received, 1)
b.callback(True)
# assert bad results - should fail
bad_output = output.parent / "badfile.result"
ctd = task.query_extra_data(10000.).ctd
task.computation_finished(ctd['subtask_id'], [str(bad_output)],
verification_finished=failure)
reactor.iterate()
sync_wait(b, 40)
def test_pause_task(self, mock_addr):
mock_addr.return_value = self.addr_return
with self.assertLogs(logger, level="WARNING"):
assert self.tm.pause_task("xyz") is None
t = self._get_task_mock()
sync_wait(self.tm.add_new_task(t))
with self.assertNoLogs(logger, level="WARNING"):
self.tm.pause_task("xyz")
assert self.tm.tasks["xyz"].task_status == TaskStatus.paused
assert self.tm.tasks_states["xyz"].status == TaskStatus.paused
def test_query_task_state(self, mock_addr):
mock_addr.return_value = self.addr_return
with self.assertLogs(logger, level="WARNING"):
assert self.tm.query_task_state("xyz") is None
t = self._get_task_mock()
sync_wait(self.tm.add_new_task(t))
with self.assertNoLogs(logger, level="WARNING"):
ts = self.tm.query_task_state("xyz")
assert ts is not None
assert ts.progress == 0.3
def load(self, file_name, skip_test):
try:
definition = self.__read_from_file(file_name)
except Exception as exc:
return CommandResult(
error="Error reading task from file '{}': {}".format(
file_name, exc))
if hasattr(definition, 'resources'):
definition.resources = {
os.path.normpath(res)
for res in definition.resources
}
datadir = sync_wait(Tasks.client.get_datadir())
# TODO: unify GUI and CLI logic
rendering_task_state = TaskDesc()
rendering_task_state.definition = definition
rendering_task_state.task_state.status = TaskStatus.starting
if not Tasks.application_logic:
Tasks.application_logic = CommandAppLogic.instantiate(
Tasks.client, datadir)
task_builder = Tasks.application_logic.get_builder(
rendering_task_state)
task = Task.build_task(task_builder)
rendering_task_state.task_state.outputs = task.get_output_names()
rendering_task_state.task_state.total_subtasks = task.get_total_tasks()
task.header.task_id = str(uuid.uuid4())
if not skip_test:
test_task = Task.build_task(task_builder)
test_task.header.task_id = str(uuid.uuid4())
queue = Queue()
TaskTester(test_task,
datadir,
success_callback=lambda *a, **kw: queue.put(True),
error_callback=lambda *a, **kw: queue.put(a)).run()
test_result = queue.get()
if test_result is not True:
return CommandResult(
error="Test failed: {}".format(test_result))
task_dict = DictSerializer.dump(task)
task_def = task_dict['task_definition']
task_def['resources'] = list(task_def.get('task_definition', []))
deferred = Tasks.client.create_task(task_dict)
return sync_wait(deferred, timeout=1800)
def test_change_timeouts(self, mock_addr):
mock_addr.return_value = self.addr_return
t = self._get_task_mock(timeout=20, subtask_timeout=40)
sync_wait(self.tm.add_new_task(t))
assert get_timestamp_utc() + 15 <= t.header.deadline
assert t.header.deadline <= get_timestamp_utc() + 20
assert t.header.subtask_timeout == 40
self.tm.change_timeouts("xyz", 60, 10)
assert get_timestamp_utc() + 55 <= t.header.deadline
assert t.header.deadline <= get_timestamp_utc() + 60
assert t.header.subtask_timeout == 10
def test_run_benchmark_fail(self, *_):
from apps.dummy.dummyenvironment import DummyTaskEnvironment
def raise_exc(*_args, **_kwargs):
raise Exception('Test exception')
with patch("golem.docker.image.DockerImage.is_available",
return_value=True), \
patch("golem.docker.job.DockerJob.__init__",
side_effect=raise_exc), \
self.assertRaisesRegex(Exception, 'Test exception'):
sync_wait(self.client.run_benchmark(DummyTaskEnvironment.get_id()))
def clear(self, provider, requestor):
if not provider and not requestor:
return CommandResult(error="Target role was not specified "
"(provider / requestor)")
clear = Resources.client.clear_dir
if requestor:
return sync_wait(clear(DirectoryType.RECEIVED), timeout=None)
elif provider:
return sync_wait(clear(DirectoryType.DISTRIBUTED), timeout=None)
def __init__(self, task_server: 'TaskServer', use_docker_manager=True,
finished_cb=None) -> None:
self.task_server = task_server
# Id of the task that we're currently waiting for for
self.waiting_for_task: Optional[str] = None
# Id of the task that we're currently computing
self.counting_task = None
# TaskThread
self.counting_thread = None
self.task_requested = False
# Is task computer currently able to run computation?
self.runnable = True
self.listeners = []
self.last_task_request = time.time()
# when we should stop waiting for the task
self.waiting_deadline = None
self.dir_manager = None
self.resource_manager: Optional[ResourcesManager] = None
self.task_request_frequency = None
# Is there a time limit after which we don't wait for task timeout
# anymore
self.use_waiting_deadline = False
self.waiting_for_task_session_timeout = None
self.docker_manager: DockerManager = DockerManager.install()
if use_docker_manager:
self.docker_manager.check_environment()
self.use_docker_manager = use_docker_manager
run_benchmarks = self.task_server.benchmark_manager.benchmarks_needed()
deferred = self.change_config(
task_server.config_desc, in_background=False,
run_benchmarks=run_benchmarks)
try:
sync_wait(deferred, BENCHMARK_TIMEOUT)
except TimeoutError:
logger.warning('Benchmark computation timed out')
self.stats = IntStatsKeeper(CompStats)
self.assigned_subtask: Optional[Dict[str, Any]] = None
self.max_assigned_tasks = 1
self.delta = None
self.last_task_timeout_checking = None
self.support_direct_computation = False
# Should this node behave as provider and compute tasks?
self.compute_tasks = task_server.config_desc.accept_tasks \
and not task_server.config_desc.in_shutdown
self.finished_cb = finished_cb
def test_run_test_task_params(self, *_):
with mock.patch(
'apps.blender.task.blenderrendertask.'
'BlenderTaskTypeInfo.for_purpose',
),\
mock.patch('golem.client.TaskTester.run'):
golem_deferred.sync_wait(
rpc._run_test_task(
self.client, {
'type': 'blender',
'resources': ['_.blend'],
'subtasks_count': 1,
}))
def test_wait_for_deferred(self):
self.assertEqual(sync_wait('1234'), '1234')
deferred = Deferred()
deferred.result = '5678'
deferred.called = True
self.assertEqual(sync_wait(deferred), '5678')
with self.assertRaises(TimeoutError):
deferred_2 = Deferred()
sync_wait(deferred_2, timeout=0)
def test_ensure_task_deposit(self, *_):
force = fake.pybool()
self.client.concent_service = mock.Mock()
self.client.concent_service.enabled = True
self.t_dict['concent_enabled'] = True
task = self.client.task_manager.create_task(self.t_dict)
deferred = rpc.enqueue_new_task(self.client, task, force=force)
golem_deferred.sync_wait(deferred)
self.client.transaction_system.concent_deposit.assert_called_once_with(
required=mock.ANY,
expected=mock.ANY,
force=force,
)
def unlock(self) -> str: # pylint: disable=no-self-use
from twisted.internet import threads
client = Account.client
is_account_unlocked: bool = sync_wait(client.is_account_unlocked())
if is_account_unlocked:
return "Account already unlocked"
has_key = sync_wait(client.key_exists())
if not has_key:
print("No account found, generate one by setting a password")
else:
print("Unlock your account to start golem")
print("This command will time out in 30 seconds.")
defer_getpass = threads.deferToThread(getpass.getpass, 'Password:'******''
result = zxcvbn.zxcvbn(pswd, user_inputs=['Golem', account_name])
# print(result['score'])
if result['score'] < MIN_SCORE:
return "Password is not strong enough. " \
"Please use capitals, numbers and special characters."
# Confirm the password
confirm = getpass.getpass('Confirm password:')
if confirm != pswd:
return "Password and confirmation do not match."
print("Generating keys, this can take up to 10 minutes...")
success = sync_wait(client.set_password(pswd), timeout=15 * 60)
if not success:
return "Incorrect password"
return "Account unlock success"
def test_thread(self):
ts = mock.MagicMock()
ts.config_desc = ClientConfigDescriptor()
ts.benchmark_manager.benchmarks_needed.return_value = False
ts.get_task_computer_root.return_value = self.new_path
tc = TaskComputer(ts, use_docker_manager=False)
tc.counting_task = True
tc.waiting_for_task = None
tt = self._new_task_thread(tc)
sync_wait(tt.start())
self.assertGreater(tt.end_time - tt.start_time, 0)
self.assertLess(tt.end_time - tt.start_time, 20)
def show(self, basic, provider, requestor):
config = sync_wait(Settings.client.get_settings())
if not (basic ^ provider) and not (provider ^ requestor):
return config
result = dict()
if basic:
result.update({
k: v
for k, v in config.iteritems() if k in Settings.basic_settings
})
if requestor:
result.update({
k: v
for k, v in config.iteritems()
if k in Settings.requestor_settings
})
if provider:
result.update({
k: v
for k, v in config.iteritems()
if k not in Settings.basic_settings
and k not in Settings.requestor_settings
})
return result
def payments(sort, status):
deferred = payments.client.get_payments_list()
result = sync_wait(deferred) or []
values = []
if status is not None:
result = filter_by_status(result, status)
for payment in result:
payment_value = float(payment["value"])
payment_fee = payment["fee"] or ""
if payment_fee:
payment_fee = __value(payment_fee, "ETH")
entry = [
to_unicode(payment["subtask"]),
to_unicode(payment["payee"]),
to_unicode(payment["status"]),
__value(payment_value, "GNT"),
to_unicode(payment_fee)
]
values.append(entry)
return CommandResult.to_tabular(payments_table_headers, values, sort=sort)
def payments(sort):
deferred = payments.client.get_payments_list()
result = sync_wait(deferred) or []
values = []
for payment in result:
payment_value = float(payment["value"])
payment_fee = payment["fee"] or u""
if payment_fee:
payment_fee = u"{:.1f}%".format(float(payment_fee) * 100 /
payment_value)
entry = [
payment["subtask"],
payment["payee"].encode('hex'),
__status(payment),
__value(payment_value),
payment_fee
]
values.append(entry)
return CommandResult.to_tabular(payments_table_headers, values, sort=sort)
def test_locked(self, *_):
self.client.concent_service.variant = CONCENT_CHOICES['test']
self.assertTrue(self.client.concent_service.enabled)
self.client.transaction_system.concent_timelock\
.return_value = 0
result = sync_wait(self.client.get_deposit_balance())
self.assertEqual(result['status'], 'locked')
def test_get_balance(self, *_):
c = self.client
c.transaction_system = Mock()
result = {
'gnt_available': 2,
'gnt_locked': 1,
'gnt_nonconverted': 0,
'gnt_update_time': None,
'eth_available': 2,
'eth_locked': 1,
'eth_update_time': None,
'block_number': 222,
}
c.transaction_system.get_balance.return_value = result
balance = sync_wait(c.get_balance())
assert balance == {
'gnt': "2",
'av_gnt': "2",
'eth': "2",
'gnt_nonconverted': "0",
'gnt_lock': "1",
'eth_lock': "1",
'last_gnt_update': "None",
'last_eth_update': "None",
'block_number': "222",
}
assert all(isinstance(entry, str) for entry in balance)
def show(self, id, sort, current):
deferred = Tasks.client.get_tasks(id)
result = sync_wait(deferred)
if not id:
values = []
if current:
result = [
t for t in result if TaskStatus(t['status']).is_active()
]
for task in result:
values.append([
task['id'],
Tasks.__format_seconds(task['time_remaining']),
str(task['subtasks_count']), task['status'],
Tasks.__progress_str(task['progress'])
])
return CommandResult.to_tabular(Tasks.task_table_headers,
values,
sort=sort)
if isinstance(result, dict):
result['time_remaining'] = \
Tasks.__format_seconds(result['time_remaining'])
result['progress'] = Tasks.__progress_str(result['progress'])
return result
def test_get_resources(self, mock_addr):
mock_addr.return_value = self.addr_return
task_id = "xyz"
resources = ['first', 'second']
task_mock = self._get_task_mock()
with patch('golem.task.taskmanager.TaskManager.get_resources',
return_value=resources):
sync_wait(self.tm.add_new_task(task_mock))
assert self.tm.get_resources(task_id,
task_mock.header) is resources
task = Task(self._get_task_header("xyz", 120, 120),
"print 'hello world'")
self.tm.tasks["xyz"] = task
self.tm.get_resources("xyz", TaskResourceHeader(self.path), 0)
def test_run_test_task_error(self, *_):
error = ('error', 'error')
more = {'more': 'more'}
def _run(_self: tasktester.TaskTester):
self._check_task_tester_result()
_self.error_callback(*error, **more)
with mock.patch('golem.client.TaskTester.run', _run):
golem_deferred.sync_wait(rpc._run_test_task(self.client, {}))
self.assertIsInstance(self.client.task_test_result, dict)
self.assertEqual(self.client.task_test_result, {
"status": taskstate.TaskTestStatus.error,
"error": error,
"more": more
})