def test_is_supported(self):
em = EnvironmentsManager()
em.environments = {}
em.support_statuses = {}
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10.0)
header = get_task_header()
header.fixed_header.environment = None
header.fixed_header.max_price = None
header.fixed_header.min_version = None
self.assertFalse(tk.check_support(header))
header.fixed_header.environment = Environment.get_id()
header.fixed_header.max_price = 0
supported = tk.check_support(header)
self.assertFalse(supported)
self.assertIn(UnsupportReason.ENVIRONMENT_MISSING, supported.desc)
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
supported = tk.check_support(header)
self.assertFalse(supported)
self.assertIn(UnsupportReason.MAX_PRICE, supported.desc)
header.fixed_header.max_price = 10.0
supported = tk.check_support(header)
self.assertFalse(supported)
self.assertIn(UnsupportReason.APP_VERSION, supported.desc)
header.fixed_header.min_version = golem.__version__
self.assertTrue(tk.check_support(header))
header.fixed_header.max_price = 10.0
self.assertTrue(tk.check_support(header))
config_desc = mock.Mock()
config_desc.min_price = 13.0
tk.change_config(config_desc)
self.assertFalse(tk.check_support(header))
config_desc.min_price = 10.0
tk.change_config(config_desc)
self.assertTrue(tk.check_support(header))
header.fixed_header.min_version = "120"
self.assertFalse(tk.check_support(header))
header.fixed_header.min_version = tk.app_version
self.assertTrue(tk.check_support(header))
header.fixed_header.min_version = "abc"
with self.assertLogs(logger=logger, level='WARNING'):
self.assertFalse(tk.check_support(header))
def test_is_supported(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10.0)
self.assertFalse(tk.is_supported({}))
task = {"environment": Environment.get_id(), 'max_price': 0}
self.assertFalse(tk.is_supported(task))
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
self.assertFalse(tk.is_supported(task))
task["max_price"] = 10.0
self.assertFalse(tk.is_supported(task))
task["min_version"] = APP_VERSION
self.assertTrue(tk.is_supported(task))
task["max_price"] = 10.5
self.assertTrue(tk.is_supported(task))
config_desc = Mock()
config_desc.min_price = 13.0
tk.change_config(config_desc)
self.assertFalse(tk.is_supported(task))
config_desc.min_price = 10.0
tk.change_config(config_desc)
self.assertTrue(tk.is_supported(task))
task["min_version"] = "120"
self.assertFalse(tk.is_supported(task))
task["min_version"] = tk.app_version
self.assertTrue(tk.is_supported(task))
task["min_version"] = "abc"
with self.assertLogs(logger=logger, level=1):
self.assertFalse(tk.is_supported(task))
def test_get_task(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10)
self.assertIsNone(tk.get_task())
task_header = get_dict_task_header()
task_header["task_id"] = "uvw"
self.assertTrue(tk.add_task_header(task_header))
self.assertIsNone(tk.get_task())
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
task_header["task_id"] = "xyz"
self.assertTrue(tk.add_task_header(task_header))
th = tk.get_task()
assert isinstance(th.task_owner, Node)
self.assertEqual(task_header["task_id"], th.task_id)
self.assertEqual(task_header["max_price"], th.max_price)
self.assertEqual(task_header["node_name"], th.node_name)
self.assertEqual(task_header["task_owner_port"], th.task_owner_port)
self.assertEqual(task_header["task_owner_key_id"],
th.task_owner_key_id)
self.assertEqual(task_header["environment"], th.environment)
self.assertEqual(task_header["deadline"], th.deadline)
self.assertEqual(task_header["subtask_timeout"], th.subtask_timeout)
self.assertEqual(task_header["max_price"], th.max_price)
self.assertEqual(task_header["task_id"], th.task_id)
def test_check_version_compatibility(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10.0)
tk.app_version = '0.4.5'
with self.assertRaises(ValueError):
tk.check_version_compatibility('')
with self.assertRaises(ValueError):
tk.check_version_compatibility('0')
with self.assertRaises(ValueError):
tk.check_version_compatibility('1.5')
with self.assertRaises(ValueError):
tk.check_version_compatibility('0.4-alpha+build.2004.01.01')
with self.assertRaises(ValueError):
tk.check_version_compatibility('0.4-alpha')
with self.assertRaises(ValueError):
tk.check_version_compatibility('0.4-alpha')
assert not tk.check_version_compatibility('1.5.0')
assert not tk.check_version_compatibility('1.4.0')
assert not tk.check_version_compatibility('0.5.0')
assert not tk.check_version_compatibility('0.4.6')
assert not tk.check_version_compatibility('0.3.0')
assert tk.check_version_compatibility('0.4.5')
assert tk.check_version_compatibility('0.4.1')
assert tk.check_version_compatibility('0.4.0')
assert tk.check_version_compatibility('0.4.0-alpha')
assert tk.check_version_compatibility('0.4.0-alpha+build')
assert tk.check_version_compatibility('0.4.0-alpha+build.2010')
def test_task_header_update(self):
e = Environment()
e.accept_tasks = True
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10)
tk.environments_manager.add_environment(e)
task_header = get_task_header()
task_id = task_header.task_id
task_header.fixed_header.deadline = timeout_to_deadline(10)
task_header.fixed_header.update_checksum()
assert tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
assert tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
task_header = copy.deepcopy(task_header)
task_header.fixed_header.max_price = 1
task_header.fixed_header.update_checksum()
# An attempt to update fixed header should *not* succeed
assert not tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
tk.task_headers = {}
tk.supported_tasks = []
assert tk.add_task_header(task_header)
assert task_id not in tk.supported_tasks
def test_change_config(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10.0)
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
task_header = get_dict_task_header()
task_header["max_price"] = 9.0
tk.add_task_header(task_header)
self.assertNotIn("xyz", tk.supported_tasks)
self.assertIsNotNone(tk.task_headers["xyz"])
task_header["task_id"] = "abc"
task_header["max_price"] = 10.0
tk.add_task_header(task_header)
self.assertIn("abc", tk.supported_tasks)
self.assertIsNotNone(tk.task_headers["abc"])
config_desc = Mock()
config_desc.min_price = 10.0
tk.change_config(config_desc)
self.assertNotIn("xyz", tk.supported_tasks)
self.assertIn("abc", tk.supported_tasks)
config_desc.min_price = 8.0
tk.change_config(config_desc)
self.assertIn("xyz", tk.supported_tasks)
self.assertIn("abc", tk.supported_tasks)
config_desc.min_price = 11.0
tk.change_config(config_desc)
self.assertNotIn("xyz", tk.supported_tasks)
self.assertNotIn("abc", tk.supported_tasks)
def test_old_tasks(frozen_time, _): # pylint: disable=no-self-argument
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10)
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
task_header = get_task_header()
task_header.fixed_header.deadline = timeout_to_deadline(10)
assert tk.add_task_header(task_header)
task_id = task_header.task_id
task_header2 = get_task_header("abc")
task_header2.fixed_header.deadline = timeout_to_deadline(1)
task_id2 = task_header2.task_id
assert tk.add_task_header(task_header2)
assert tk.task_headers.get(task_id2) is not None
assert tk.task_headers.get(task_id) is not None
assert tk.removed_tasks.get(task_id2) is None
assert tk.removed_tasks.get(task_id) is None
assert len(tk.supported_tasks) == 2
frozen_time.tick(timedelta(seconds=1.1)) # pylint: disable=no-member
tk.remove_old_tasks()
assert tk.task_headers.get(task_id2) is None
assert tk.task_headers.get(task_id) is not None
assert tk.removed_tasks.get(task_id2) is not None
assert tk.removed_tasks.get(task_id) is None
assert len(tk.supported_tasks) == 1
assert tk.supported_tasks[0] == task_id
def test_is_correct(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10)
th = get_dict_task_header()
correct, err = tk.is_correct(th)
assert correct
assert err is None
th['deadline'] = datetime.now()
correct, err = tk.is_correct(th)
assert not correct
assert err == "Deadline is not a timestamp"
th['deadline'] = get_timestamp_utc() - 10
correct, err = tk.is_correct(th)
assert not correct
assert err == "Deadline already passed"
th['deadline'] = get_timestamp_utc() + 20
correct, err = tk.is_correct(th)
assert correct
assert err is None
th['subtask_timeout'] = "abc"
correct, err = tk.is_correct(th)
assert not correct
assert err == "Subtask timeout is not a number"
th['subtask_timeout'] = -131
correct, err = tk.is_correct(th)
assert not correct
assert err == "Subtask timeout is less than 0"
def test_task_header_update(self):
e = Environment()
e.accept_tasks = True
tk = TaskHeaderKeeper(EnvironmentsManager(), 10)
tk.environments_manager.add_environment(e)
assert not tk.add_task_header(dict())
task_header = get_dict_task_header()
task_id = task_header["task_id"]
task_header["deadline"] = timeout_to_deadline(10)
assert tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
assert tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
task_header["max_price"] = 1
assert tk.add_task_header(task_header)
assert task_id not in tk.supported_tasks
tk.task_headers = {}
tk.supported_tasks = []
task_header["max_price"] = 1
assert tk.add_task_header(task_header)
assert task_id not in tk.supported_tasks
task_header['task_id'] = "newtaskID"
task_header['deadline'] = "WRONG DEADLINE"
assert not tk.add_task_header(task_header)
def test_get_owner(self):
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10)
header = get_task_header()
owner = header.task_owner.key
key_id = header.task_id
tk.add_task_header(header)
assert tk.get_owner(key_id) == owner
assert tk.get_owner("UNKNOWN") is None
def test_task_limit(frozen_time, self): # pylint: disable=no-self-argument
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10)
limit = tk.max_tasks_per_requestor
thd = get_task_header("ta")
thd.fixed_header.deadline = timeout_to_deadline(0.1)
tk.add_task_header(thd)
ids = [thd.task_id]
for i in range(1, limit):
thd = get_task_header("ta")
ids.append(thd.task_id)
tk.add_task_header(thd)
last_add_time = time.time()
for id_ in ids:
self.assertIn(id_, tk.task_headers)
thd = get_task_header("tb0")
tb_id = thd.task_id
tk.add_task_header(thd)
for id_ in ids:
self.assertIn(id_, tk.task_headers)
self.assertIn(tb_id, tk.task_headers)
while time.time() == last_add_time:
frozen_time.tick( # pylint: disable=no-member
delta=timedelta(milliseconds=100))
thd = get_task_header("ta")
new_task_id = thd.task_id
tk.add_task_header(thd)
self.assertNotIn(new_task_id, tk.task_headers)
for id_ in ids:
self.assertIn(id_, tk.task_headers)
self.assertIn(tb_id, tk.task_headers)
frozen_time.tick( # pylint: disable=no-member
delta=timedelta(milliseconds=100))
tk.remove_old_tasks()
thd = get_task_header("ta")
new_task_id = thd.task_id
tk.add_task_header(thd)
self.assertIn(new_task_id, tk.task_headers)
self.assertNotIn(ids[0], tk.task_headers)
for i in range(1, limit):
self.assertIn(ids[i], tk.task_headers)
self.assertIn(tb_id, tk.task_headers)
def test_check_max_tasks_per_owner(self):
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10,
max_tasks_per_requestor=10)
limit = tk.max_tasks_per_requestor
new_limit = 3
ids = []
for i in range(new_limit):
thd = get_task_header("ta")
ids.append(thd.task_id)
tk.add_task_header(thd)
last_add_time = time.time()
thd = get_task_header("tb0")
tb0_id = thd.task_id
tk.add_task_header(thd)
for id_ in ids:
self.assertIn(id_, tk.task_headers)
self.assertIn(tb0_id, tk.task_headers)
while time.time() == last_add_time:
time.sleep(0.1)
new_ids = []
for i in range(new_limit, limit):
thd = get_task_header("ta")
new_ids.append(thd.task_id)
tk.add_task_header(thd)
for id_ in ids + new_ids:
self.assertIn(id_, tk.task_headers)
self.assertIn(tb0_id, tk.task_headers)
self.assertEqual(limit + 1, len(tk.task_headers))
# shouldn't remove any tasks
tk.check_max_tasks_per_owner(thd.task_owner.key)
for id_ in ids + new_ids:
self.assertIn(id_, tk.task_headers)
self.assertIn(tb0_id, tk.task_headers)
self.assertEqual(limit + 1, len(tk.task_headers))
tk.max_tasks_per_requestor = new_limit
# should remove ta{3..9}
tk.check_max_tasks_per_owner(thd.task_owner.key)
for id_ in ids:
self.assertIn(id_, tk.task_headers)
self.assertIn(tb0_id, tk.task_headers)
self.assertEqual(new_limit + 1, len(tk.task_headers))
def test_get_task(self):
em = EnvironmentsManager()
em.environments = {}
em.support_statuses = {}
tk = TaskHeaderKeeper(environments_manager=em,
node=p2p.Node(),
min_price=10)
self.assertIsNone(tk.get_task())
task_header = get_task_header("uvw")
self.assertTrue(tk.add_task_header(task_header))
self.assertIsNone(tk.get_task())
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
task_header2 = get_task_header("xyz")
self.assertTrue(tk.add_task_header(task_header2))
th = tk.get_task()
assert isinstance(th.task_owner, Node)
self.assertEqual(task_header2.to_dict(), th.to_dict())
def test_get_environment_by_id(self):
em = EnvironmentsManager()
env1 = Environment()
env2 = Environment()
env3 = Environment()
env1.get_id = lambda: "Env1"
env2.get_id = lambda: "Env2"
env3.get_id = lambda: "Env3"
em.add_environment(env1)
em.add_environment(env2)
em.add_environment(env3)
self.assertTrue(env1 == em.get_environment_by_id("Env1"))
self.assertTrue(env2 == em.get_environment_by_id("Env2"))
self.assertTrue(env3 == em.get_environment_by_id("Env3"))
def test_change_config(self, tar):
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10.0,
task_archiver=tar)
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
task_header = get_task_header()
task_id = task_header.task_id
task_header.fixed_header.max_price = 9.0
tk.add_task_header(task_header)
self.assertNotIn(task_id, tk.supported_tasks)
self.assertIn(task_id, tk.task_headers)
task_header = get_task_header("abc")
task_id2 = task_header.task_id
task_header.fixed_header.max_price = 10.0
tk.add_task_header(task_header)
self.assertIn(task_id2, tk.supported_tasks)
self.assertIn(task_id2, tk.task_headers)
config_desc = mock.Mock()
config_desc.min_price = 10.0
tk.change_config(config_desc)
self.assertNotIn(task_id, tk.supported_tasks)
self.assertIn(task_id2, tk.supported_tasks)
config_desc.min_price = 8.0
tk.change_config(config_desc)
self.assertIn(task_id, tk.supported_tasks)
self.assertIn(task_id2, tk.supported_tasks)
config_desc.min_price = 11.0
tk.change_config(config_desc)
self.assertNotIn(task_id, tk.supported_tasks)
self.assertNotIn(task_id2, tk.supported_tasks)
# Make sure the tasks stats are properly archived
tar.reset_mock()
config_desc.min_price = 9.5
tk.change_config(config_desc)
self.assertNotIn(task_id, tk.supported_tasks)
self.assertIn(task_id2, tk.supported_tasks)
tar.add_support_status.assert_any_call(
task_id, SupportStatus(False, {UnsupportReason.MAX_PRICE: 9.0}))
tar.add_support_status.assert_any_call(task_id2,
SupportStatus(True, {}))
def test_check_version_compatibility(self):
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10.0)
tk.app_version = '0.4.5-dev+232.138018'
for v in ['', '0', '1.5', '0.4-alpha+build.2004.01.01', '0.4-alpha']:
with self.assertRaises(ValueError, msg=v):
tk.check_version_compatibility(v)
for v in ['1.5.0', '1.4.0', '0.5.0', '0.3.0']:
self.assertFalse(tk.check_version_compatibility(v), msg=v)
for v in [
'0.4.5', '0.4.1', '0.4.0', '0.4.0-alpha', '0.4.0-alpha+build',
'0.4.0-alpha+build.2010', '0.4.6'
]:
self.assertTrue(tk.check_version_compatibility(v), msg=v)
def test_task_header_update_stats(self, tar):
e = Environment()
e.accept_tasks = True
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10,
task_archiver=tar)
tk.environments_manager.add_environment(e)
task_header = get_task_header("good")
assert tk.add_task_header(task_header)
tar.add_task.assert_called_with(mock.ANY)
task_id = task_header.task_id
tar.add_support_status.assert_any_call(task_id,
SupportStatus(True, {}))
tar.reset_mock()
task_header2 = get_task_header("bad")
task_id2 = task_header2.task_id
task_header2.fixed_header.max_price = 1.0
assert tk.add_task_header(task_header2)
tar.add_task.assert_called_with(mock.ANY)
tar.add_support_status.assert_any_call(
task_id2, SupportStatus(False, {UnsupportReason.MAX_PRICE: 1.0}))
def test_old_tasks(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10)
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
task_header = get_dict_task_header()
task_header["deadline"] = timeout_to_deadline(10)
assert tk.add_task_header(task_header)
task_header["deadline"] = timeout_to_deadline(1)
task_header["task_id"] = "abc"
assert tk.add_task_header(task_header)
assert tk.task_headers.get("abc") is not None
assert tk.task_headers.get("xyz") is not None
assert tk.removed_tasks.get("abc") is None
assert tk.removed_tasks.get("xyz") is None
assert len(tk.supported_tasks) == 2
time.sleep(1.1)
tk.remove_old_tasks()
assert tk.task_headers.get("abc") is None
assert tk.task_headers.get("xyz") is not None
assert tk.removed_tasks.get("abc") is not None
assert tk.removed_tasks.get("xyz") is None
assert len(tk.supported_tasks) == 1
assert tk.supported_tasks[0] == "xyz"
def test_load_config_manager_empty(self):
# FIXME: Passing empty env list breaks config write.
mgr = EnvironmentsManager()
mgr.load_config(self.path)
assert mgr.env_config
def test_load_config_manager(self):
mgr = EnvironmentsManager()
mgr.environments.add(Environment())
mgr.load_config(self.path)
assert mgr.env_config
def _run_docker_job(self) -> Optional[int]:
self.dir_mapping.mkdirs()
binds = self._get_default_binds()
volumes = list(bind.target for bind in binds)
environment = self._get_environment()
environment.update(
WORK_DIR=DockerJob.WORK_DIR,
RESOURCES_DIR=DockerJob.RESOURCES_DIR,
OUTPUT_DIR=DockerJob.OUTPUT_DIR
)
assert self.image is not None
docker_env = EnvironmentsManager().get_environment_by_image(self.image)
if docker_env:
env_config = docker_env.get_container_config()
environment.update(env_config['environment'])
binds += env_config['binds']
volumes += env_config['volumes']
devices = env_config['devices']
runtime = env_config['runtime']
else:
logger.debug('No Docker environment found for image %r', self.image)
devices = None
runtime = None
assert self.docker_manager is not None
# PyLint still thinks docker_manager is of type DockerConfigManager
# pylint: disable=no-member
host_config = self.docker_manager.get_host_config_for_task(binds)
host_config['devices'] = devices
host_config['runtime'] = runtime
params = dict(
image=self.image,
script_src=self.src_code,
parameters=self.extra_data,
resources_dir=str(self.dir_mapping.resources),
work_dir=str(self.dir_mapping.work),
output_dir=str(self.dir_mapping.output),
volumes=volumes,
environment=environment,
host_config=host_config
)
with DockerJob(**params) as job, MemoryChecker(self.check_mem) as mc:
self.job = job
job.start()
exit_code = job.wait()
estm_mem = mc.estm_mem
job.dump_logs(str(self.dir_mapping.logs / self.STDOUT_FILE),
str(self.dir_mapping.logs / self.STDERR_FILE))
if exit_code != 0:
std_err = (self.dir_mapping.logs / self.STDERR_FILE).read_text()
logger.warning(f'Task stderr:\n{std_err}')
raise JobException(self._exit_code_message(exit_code))
return estm_mem
def __init__(self,
datadir=None,
transaction_system=False,
connect_to_known_hosts=True,
use_docker_machine_manager=True,
use_monitor=True,
**config_overrides):
if not datadir:
datadir = get_local_datadir('default')
self.datadir = datadir
self.__lock_datadir()
self.lock = Lock()
self.task_tester = None
# Read and validate configuration
config = AppConfig.load_config(datadir)
self.config_desc = ClientConfigDescriptor()
self.config_desc.init_from_app_config(config)
for key, val in config_overrides.iteritems():
if not hasattr(self.config_desc, key):
self.quit() # quit only closes underlying services (for now)
raise AttributeError(
"Can't override nonexistent config entry '{}'".format(key))
setattr(self.config_desc, key, val)
self.config_approver = ConfigApprover(self.config_desc)
log.info('Client "%s", datadir: %s', self.config_desc.node_name,
datadir)
# Initialize database
self.db = Database(datadir)
# Hardware configuration
HardwarePresets.initialize(self.datadir)
HardwarePresets.update_config(self.config_desc.hardware_preset_name,
self.config_desc)
self.keys_auth = EllipticalKeysAuth(self.datadir)
# NETWORK
self.node = Node(node_name=self.config_desc.node_name,
prv_addr=self.config_desc.node_address,
key=self.keys_auth.get_key_id())
self.p2pservice = None
self.diag_service = None
self.task_server = None
self.last_nss_time = time.time()
self.last_net_check_time = time.time()
self.last_balance_time = time.time()
self.last_tasks_time = time.time()
self.last_node_state_snapshot = None
self.nodes_manager_client = None
self.do_work_task = task.LoopingCall(self.__do_work)
self.publish_task = task.LoopingCall(self.__publish_events)
self.cfg = config
self.send_snapshot = False
self.snapshot_lock = Lock()
self.ranking = Ranking(self)
if transaction_system:
# Bootstrap transaction system if enabled.
# TODO: Transaction system (and possible other modules) should be
# modeled as a Service that run independently.
# The Client/Application should be a collection of services.
self.transaction_system = EthereumTransactionSystem(
datadir, self.keys_auth._private_key)
else:
self.transaction_system = None
self.use_docker_machine_manager = use_docker_machine_manager
self.connect_to_known_hosts = connect_to_known_hosts
self.environments_manager = EnvironmentsManager()
self.daemon_manager = None
self.rpc_publisher = None
self.ipfs_manager = None
self.resource_server = None
self.resource_port = 0
self.last_get_resource_peers_time = time.time()
self.get_resource_peers_interval = 5.0
self.use_monitor = use_monitor
self.monitor = None
self.session_id = uuid.uuid4().get_hex()
dispatcher.connect(self.p2p_listener, signal='golem.p2p')
dispatcher.connect(self.taskmanager_listener,
signal='golem.taskmanager')
atexit.register(self.quit)
class Client(HardwarePresetsMixin):
def __init__(self,
datadir=None,
transaction_system=False,
connect_to_known_hosts=True,
use_docker_machine_manager=True,
use_monitor=True,
**config_overrides):
if not datadir:
datadir = get_local_datadir('default')
self.datadir = datadir
self.__lock_datadir()
self.lock = Lock()
self.task_tester = None
# Read and validate configuration
config = AppConfig.load_config(datadir)
self.config_desc = ClientConfigDescriptor()
self.config_desc.init_from_app_config(config)
for key, val in config_overrides.iteritems():
if not hasattr(self.config_desc, key):
self.quit() # quit only closes underlying services (for now)
raise AttributeError(
"Can't override nonexistent config entry '{}'".format(key))
setattr(self.config_desc, key, val)
self.config_approver = ConfigApprover(self.config_desc)
log.info('Client "%s", datadir: %s', self.config_desc.node_name,
datadir)
# Initialize database
self.db = Database(datadir)
# Hardware configuration
HardwarePresets.initialize(self.datadir)
HardwarePresets.update_config(self.config_desc.hardware_preset_name,
self.config_desc)
self.keys_auth = EllipticalKeysAuth(self.datadir)
# NETWORK
self.node = Node(node_name=self.config_desc.node_name,
prv_addr=self.config_desc.node_address,
key=self.keys_auth.get_key_id())
self.p2pservice = None
self.diag_service = None
self.task_server = None
self.last_nss_time = time.time()
self.last_net_check_time = time.time()
self.last_balance_time = time.time()
self.last_tasks_time = time.time()
self.last_node_state_snapshot = None
self.nodes_manager_client = None
self.do_work_task = task.LoopingCall(self.__do_work)
self.publish_task = task.LoopingCall(self.__publish_events)
self.cfg = config
self.send_snapshot = False
self.snapshot_lock = Lock()
self.ranking = Ranking(self)
if transaction_system:
# Bootstrap transaction system if enabled.
# TODO: Transaction system (and possible other modules) should be
# modeled as a Service that run independently.
# The Client/Application should be a collection of services.
self.transaction_system = EthereumTransactionSystem(
datadir, self.keys_auth._private_key)
else:
self.transaction_system = None
self.use_docker_machine_manager = use_docker_machine_manager
self.connect_to_known_hosts = connect_to_known_hosts
self.environments_manager = EnvironmentsManager()
self.daemon_manager = None
self.rpc_publisher = None
self.ipfs_manager = None
self.resource_server = None
self.resource_port = 0
self.last_get_resource_peers_time = time.time()
self.get_resource_peers_interval = 5.0
self.use_monitor = use_monitor
self.monitor = None
self.session_id = uuid.uuid4().get_hex()
dispatcher.connect(self.p2p_listener, signal='golem.p2p')
dispatcher.connect(self.taskmanager_listener,
signal='golem.taskmanager')
atexit.register(self.quit)
def configure_rpc(self, rpc_session):
self.rpc_publisher = Publisher(rpc_session)
def p2p_listener(self, sender, signal, event='default', **kwargs):
if event != 'unreachable':
return
self.node.port_status = kwargs.get('description', u'')
def taskmanager_listener(self, sender, signal, event='default', **kwargs):
if event != 'task_status_updated':
return
self._publish(Task.evt_task_status, kwargs['task_id'])
# TODO: re-enable
def sync(self):
pass
def start(self):
if self.use_monitor:
self.init_monitor()
try:
self.start_network()
except SystemExit:
raise
except Exception:
log.critical('Can\'t start network. Giving up.', exc_info=True)
sys.exit(1)
self.do_work_task.start(1, False)
self.publish_task.start(1, True)
def start_network(self):
log.info("Starting network ...")
self.node.collect_network_info(self.config_desc.seed_host,
use_ipv6=self.config_desc.use_ipv6)
log.debug("Is super node? %s", self.node.is_super_node())
# self.ipfs_manager = IPFSDaemonManager(
# connect_to_bootstrap_nodes=self.connect_to_known_hosts)
# self.ipfs_manager.store_client_info()
self.p2pservice = P2PService(
self.node,
self.config_desc,
self.keys_auth,
connect_to_known_hosts=self.connect_to_known_hosts)
self.task_server = TaskServer(
self.node,
self.config_desc,
self.keys_auth,
self,
use_ipv6=self.config_desc.use_ipv6,
use_docker_machine_manager=self.use_docker_machine_manager)
dir_manager = self.task_server.task_computer.dir_manager
log.info("Starting resource server ...")
self.daemon_manager = HyperdriveDaemonManager(self.datadir)
hyperdrive_ports = self.daemon_manager.start()
resource_manager = HyperdriveResourceManager(dir_manager)
self.resource_server = BaseResourceServer(resource_manager,
dir_manager, self.keys_auth,
self)
def connect((p2p_port, task_port)):
log.info('P2P server is listening on port %s', p2p_port)
log.info('Task server is listening on port %s', task_port)
dispatcher.send(signal='golem.p2p',
event='listening',
port=[p2p_port, task_port] +
list(hyperdrive_ports))
listener = ClientTaskComputerEventListener(self)
self.task_server.task_computer.register_listener(listener)
self.p2pservice.connect_to_network()
if self.monitor:
self.diag_service.register(self.p2pservice,
self.monitor.on_peer_snapshot)
self.monitor.on_login()
def terminate(*exceptions):
log.error("Golem cannot listen on ports: %s", exceptions)
self.quit()
task = Deferred()
p2p = Deferred()
gatherResults([p2p, task],
consumeErrors=True).addCallbacks(connect, terminate)
log.info("Starting p2p server ...")
self.p2pservice.task_server = self.task_server
self.p2pservice.set_resource_server(self.resource_server)
self.p2pservice.set_metadata_manager(self)
self.p2pservice.start_accepting(listening_established=p2p.callback,
listening_failure=p2p.errback)
log.info("Starting task server ...")
self.task_server.start_accepting(listening_established=task.callback,
listening_failure=task.errback)
def init_monitor(self):
metadata = self.__get_nodemetadatamodel()
self.monitor = SystemMonitor(metadata, MONITOR_CONFIG)
self.monitor.start()
self.diag_service = DiagnosticsService(DiagnosticsOutputFormat.data)
self.diag_service.register(VMDiagnosticsProvider(),
self.monitor.on_vm_snapshot)
self.diag_service.start_looping_call()
def connect(self, socket_address):
if isinstance(socket_address, Iterable):
socket_address = SocketAddress(socket_address[0],
int(socket_address[1]))
log.debug("P2pservice connecting to %s on port %s",
socket_address.address, socket_address.port)
self.p2pservice.connect(socket_address)
def quit(self):
if self.do_work_task.running:
self.do_work_task.stop()
if self.publish_task.running:
self.publish_task.stop()
if self.task_server:
self.task_server.quit()
if self.transaction_system:
self.transaction_system.stop()
if self.diag_service:
self.diag_service.unregister_all()
if self.daemon_manager:
self.daemon_manager.stop()
dispatcher.send(signal='golem.monitor', event='shutdown')
if self.db:
self.db.close()
self._unlock_datadir()
def key_changed(self):
self.node.key = self.keys_auth.get_key_id()
self.task_server.key_changed()
self.p2pservice.key_changed()
def stop_network(self):
# FIXME: Implement this method properly - send disconnect package,
# close connections etc.
self.p2pservice = None
self.task_server = None
self.nodes_manager_client = None
def enqueue_new_task(self, task):
# FIXME: remove after the new interface has been integrated with
if isinstance(task, dict):
task = self.task_server.task_manager.create_task(task)
else:
task.header.max_price = int(task.header.max_price)
resource_manager = self.resource_server.resource_manager
task_manager = self.task_server.task_manager
task_id = task.header.task_id
key_id = self.keys_auth.key_id
options = resource_manager.build_client_options(key_id)
files = task.get_resources(None, resource_types["hashes"])
deferred = self.resource_server.add_task(files, task_id, options)
deferred.addCallback(lambda _: task_manager.add_new_task(task))
return task
def task_resource_send(self, task_id):
self.task_server.task_manager.resources_send(task_id)
def task_resource_collected(self, task_id, unpack_delta=True):
self.task_server.task_computer.task_resource_collected(
task_id, unpack_delta)
def task_resource_failure(self, task_id, reason):
self.task_server.task_computer.task_resource_failure(task_id, reason)
def set_resource_port(self, resource_port):
self.resource_port = resource_port
self.p2pservice.set_resource_peer(self.node.prv_addr,
self.resource_port)
def run_test_task(self, t_dict):
if self.task_tester is None:
request = AsyncRequest(self._run_test_task, t_dict)
async_run(request)
return True
if self.rpc_publisher:
self.rpc_publisher.publish(Task.evt_task_test_status,
TaskTestStatus.error,
u"Another test is running")
return False
def _run_test_task(self, t_dict):
def on_success(*args, **kwargs):
self.task_tester = None
self._publish(Task.evt_task_test_status, TaskTestStatus.success,
*args, **kwargs)
def on_error(*args, **kwargs):
self.task_tester = None
self._publish(Task.evt_task_test_status, TaskTestStatus.error,
*args, **kwargs)
t = DictSerializer.load(t_dict)
self.task_tester = TaskTester(t, self.datadir, on_success, on_error)
self.task_tester.run()
self._publish(Task.evt_task_test_status, TaskTestStatus.started, True)
def abort_test_task(self):
with self.lock:
if self.task_tester is not None:
self.task_tester.end_comp()
return True
return False
def create_task(self, t_dict):
try:
task = DictSerializer.load(t_dict)
new_task = self.enqueue_new_task(task)
return unicode(new_task.header.task_id)
except Exception:
log.exception("Cannot create task {}".format(t_dict))
def abort_task(self, task_id):
self.task_server.task_manager.abort_task(task_id)
def restart_task(self, task_id):
self.task_server.task_manager.restart_task(task_id)
def restart_subtask(self, subtask_id):
self.task_server.task_manager.restart_subtask(subtask_id)
def pause_task(self, task_id):
self.task_server.task_manager.pause_task(task_id)
def resume_task(self, task_id):
self.task_server.task_manager.resume_task(task_id)
def delete_task(self, task_id):
self.remove_task_header(task_id)
self.remove_task(task_id)
self.task_server.task_manager.delete_task(task_id)
def get_node(self):
return DictSerializer.dump(self.node)
def get_node_name(self):
name = self.config_desc.node_name
return unicode(name) if name else u''
def get_neighbours_degree(self):
return self.p2pservice.get_peers_degree()
def get_suggested_addr(self, key_id):
return self.p2pservice.suggested_address.get(key_id)
def get_suggested_conn_reverse(self, key_id):
return self.p2pservice.get_suggested_conn_reverse(key_id)
def get_resource_peers(self):
self.p2pservice.send_get_resource_peers()
def get_peers(self):
return self.p2pservice.peers.values()
def get_known_peers(self):
peers = self.p2pservice.free_peers or []
return [
DictSerializer.dump(PeerSessionInfo(p), typed=False) for p in peers
]
def get_connected_peers(self):
peers = self.get_peers() or []
return [
DictSerializer.dump(PeerSessionInfo(p), typed=False) for p in peers
]
def get_public_key(self):
return self.keys_auth.public_key
def get_dir_manager(self):
if self.task_server:
return self.task_server.task_computer.dir_manager
def load_keys_from_file(self, file_name):
if file_name != "":
return self.keys_auth.load_from_file(file_name)
return False
def save_keys_to_files(self, private_key_path, public_key_path):
return self.keys_auth.save_to_files(private_key_path, public_key_path)
def get_key_id(self):
return self.get_client_id()
def get_difficulty(self):
return self.keys_auth.get_difficulty()
def get_client_id(self):
key_id = self.keys_auth.get_key_id()
return unicode(key_id) if key_id else None
def get_node_key(self):
key = self.node.key
return unicode(key) if key else None
def get_settings(self):
return DictSerializer.dump(self.config_desc)
def get_setting(self, key):
if not hasattr(self.config_desc, key):
raise KeyError(u"Unknown setting: {}".format(key))
value = getattr(self.config_desc, key)
if key in ConfigApprover.numeric_opt:
return unicode(value)
return value
def update_setting(self, key, value):
if not hasattr(self.config_desc, key):
raise KeyError(u"Unknown setting: {}".format(key))
setattr(self.config_desc, key, value)
self.change_config(self.config_desc)
def update_settings(self, settings_dict, run_benchmarks=False):
cfg_desc = DictSerializer.load(settings_dict)
self.change_config(cfg_desc, run_benchmarks)
def get_datadir(self):
return unicode(self.datadir)
def get_p2p_port(self):
return self.p2pservice.cur_port
def get_task_server_port(self):
return self.task_server.cur_port
def get_task_count(self):
return len(self.task_server.task_keeper.get_all_tasks())
def get_task(self, task_id):
return self.task_server.task_manager.get_task_dict(task_id)
def get_tasks(self, task_id=None):
if task_id:
return self.task_server.task_manager.get_task_dict(task_id)
return self.task_server.task_manager.get_tasks_dict()
def get_subtasks(self, task_id):
return self.task_server.task_manager.get_subtasks_dict(task_id)
def get_subtasks_borders(self, task_id):
return self.task_server.task_manager.get_subtasks_borders(task_id)
def get_subtask(self, subtask_id):
return self.task_server.task_manager.get_subtask_dict(subtask_id)
def get_task_preview(self, task_id, single=False):
return self.task_server.task_manager.get_task_preview(task_id,
single=single)
def get_task_stats(self):
return {
u'in_network': self.get_task_count(),
u'supported': self.get_supported_task_count(),
u'subtasks_computed': self.get_computed_task_count(),
u'subtasks_with_errors': self.get_error_task_count(),
u'subtasks_with_timeout': self.get_timeout_task_count()
}
def get_supported_task_count(self):
return len(self.task_server.task_keeper.supported_tasks)
def get_computed_task_count(self):
return self.task_server.task_computer.stats.get_stats('computed_tasks')
def get_timeout_task_count(self):
return self.task_server\
.task_computer.stats.get_stats('tasks_with_timeout')
def get_error_task_count(self):
return self.task_server\
.task_computer.stats.get_stats('tasks_with_errors')
def get_payment_address(self):
address = self.transaction_system.get_payment_address()
return unicode(address) if address else None
@inlineCallbacks
def get_balance(self):
if self.use_transaction_system():
req = AsyncRequest(self.transaction_system.get_balance)
b, ab, d = yield async_run(req)
if b is not None:
returnValue((unicode(b), unicode(ab), unicode(d)))
returnValue((None, None, None))
def get_payments_list(self):
if self.use_transaction_system():
payments = self.transaction_system.get_payments_list()
return map(self._map_payment, payments)
return ()
def get_incomes_list(self):
# Will be implemented in incomes_core
return []
@classmethod
def _map_payment(cls, obj):
obj["payee"] = to_unicode(obj["payee"])
obj["value"] = to_unicode(obj["value"])
obj["fee"] = to_unicode(obj["fee"])
return obj
def get_task_cost(self, task_id):
"""
Get current cost of the task defined by @task_id
:param task_id: Task ID
:return: Cost of the task
"""
cost = self.task_server.task_manager.get_payment_for_task_id(task_id)
if cost is None:
return 0.0
return cost
def use_transaction_system(self):
return bool(self.transaction_system)
def get_computing_trust(self, node_id):
if self.use_ranking():
return self.ranking.get_computing_trust(node_id)
return None
def get_requesting_trust(self, node_id):
if self.use_ranking():
return self.ranking.get_requesting_trust(node_id)
return None
def get_description(self):
try:
account, _ = Account.get_or_create(node_id=self.get_client_id())
return account.description
except Exception as e:
return u"An error has occurred {}".format(e)
def change_description(self, description):
self.get_description()
q = Account.update(description=description)\
.where(Account.node_id == self.get_client_id())
q.execute()
def use_ranking(self):
return bool(self.ranking)
def want_to_start_task_session(self, key_id, node_id, conn_id):
self.p2pservice.want_to_start_task_session(key_id, node_id, conn_id)
def inform_about_task_nat_hole(self, key_id, rv_key_id, addr, port,
ans_conn_id):
self.p2pservice.inform_about_task_nat_hole(key_id, rv_key_id, addr,
port, ans_conn_id)
def inform_about_nat_traverse_failure(self, key_id, res_key_id, conn_id):
self.p2pservice.inform_about_nat_traverse_failure(
key_id, res_key_id, conn_id)
# CLIENT CONFIGURATION
def set_rpc_server(self, rpc_server):
self.rpc_server = rpc_server
return self.rpc_server.add_service(self)
def change_config(self, new_config_desc, run_benchmarks=False):
self.config_desc = self.config_approver.change_config(new_config_desc)
self.cfg.change_config(self.config_desc)
self.p2pservice.change_config(self.config_desc)
self.upsert_hw_preset(HardwarePresets.from_config(self.config_desc))
if self.task_server:
self.task_server.change_config(self.config_desc,
run_benchmarks=run_benchmarks)
dispatcher.send(signal='golem.monitor',
event='config_update',
meta_data=self.__get_nodemetadatamodel())
def register_nodes_manager_client(self, nodes_manager_client):
self.nodes_manager_client = nodes_manager_client
def change_timeouts(self, task_id, full_task_timeout, subtask_timeout):
self.task_server.change_timeouts(task_id, full_task_timeout,
subtask_timeout)
def query_task_state(self, task_id):
state = self.task_server.task_manager.query_task_state(task_id)
if state:
return DictSerializer.dump(state)
def pull_resources(self, task_id, resources, client_options=None):
self.resource_server.download_resources(resources,
task_id,
client_options=client_options)
def add_resource_peer(self, node_name, addr, port, key_id, node_info):
self.resource_server.add_resource_peer(node_name, addr, port, key_id,
node_info)
def get_res_dirs(self):
return {
u"computing": self.get_computed_files_dir(),
u"received": self.get_received_files_dir(),
u"distributed": self.get_distributed_files_dir()
}
def get_res_dirs_sizes(self):
return {
unicode(name): unicode(du(d))
for name, d in self.get_res_dirs().iteritems()
}
def get_res_dir(self, dir_type):
if dir_type == DirectoryType.COMPUTED:
return self.get_computed_files_dir()
elif dir_type == DirectoryType.DISTRIBUTED:
return self.get_distributed_files_dir()
elif dir_type == DirectoryType.RECEIVED:
return self.get_received_files_dir()
raise Exception(u"Unknown dir type: {}".format(dir_type))
def get_computed_files_dir(self):
return unicode(self.task_server.get_task_computer_root())
def get_received_files_dir(self):
return unicode(self.task_server.task_manager.get_task_manager_root())
def get_distributed_files_dir(self):
return unicode(self.resource_server.get_distributed_resource_root())
def clear_dir(self, dir_type):
if dir_type == DirectoryType.COMPUTED:
return self.remove_computed_files()
elif dir_type == DirectoryType.DISTRIBUTED:
return self.remove_distributed_files()
elif dir_type == DirectoryType.RECEIVED:
return self.remove_received_files()
raise Exception(u"Unknown dir type: {}".format(dir_type))
def remove_computed_files(self):
dir_manager = DirManager(self.datadir)
dir_manager.clear_dir(self.get_computed_files_dir())
def remove_distributed_files(self):
dir_manager = DirManager(self.datadir)
dir_manager.clear_dir(self.get_distributed_files_dir())
def remove_received_files(self):
dir_manager = DirManager(self.datadir)
dir_manager.clear_dir(self.get_received_files_dir())
def remove_task(self, task_id):
self.p2pservice.remove_task(task_id)
def remove_task_header(self, task_id):
self.task_server.remove_task_header(task_id)
def get_known_tasks(self):
headers = {}
for key, header in self.task_server.task_keeper.task_headers.iteritems(
): # noqa
headers[unicode(key)] = DictSerializer.dump(header)
return headers
def get_environments(self):
envs = copy(self.environments_manager.get_environments())
return [{
u'id': unicode(env.get_id()),
u'supported': env.supported(),
u'accepted': env.is_accepted(),
u'performance': env.get_performance(self.config_desc),
u'description': unicode(env.short_description)
} for env in envs]
@inlineCallbacks
def run_benchmark(self, env_id):
# TODO: move benchmarks to environments
from apps.blender.blenderenvironment import BlenderEnvironment
from apps.lux.luxenvironment import LuxRenderEnvironment
deferred = Deferred()
if env_id == BlenderEnvironment.get_id():
self.task_server.task_computer.run_blender_benchmark(
deferred.callback, deferred.errback)
elif env_id == LuxRenderEnvironment.get_id():
self.task_server.task_computer.run_lux_benchmark(
deferred.callback, deferred.errback)
else:
raise Exception("Unknown environment: {}".format(env_id))
result = yield deferred
returnValue(result)
def enable_environment(self, env_id):
self.environments_manager.change_accept_tasks(env_id, True)
def disable_environment(self, env_id):
self.environments_manager.change_accept_tasks(env_id, False)
def send_gossip(self, gossip, send_to):
return self.p2pservice.send_gossip(gossip, send_to)
def send_stop_gossip(self):
return self.p2pservice.send_stop_gossip()
def collect_gossip(self):
return self.p2pservice.pop_gossips()
def collect_stopped_peers(self):
return self.p2pservice.pop_stop_gossip_form_peers()
def collect_neighbours_loc_ranks(self):
return self.p2pservice.pop_neighbours_loc_ranks()
def push_local_rank(self, node_id, loc_rank):
self.p2pservice.push_local_rank(node_id, loc_rank)
def check_payments(self):
if not self.transaction_system:
return
after_deadline_nodes = self.transaction_system.check_payments()
for node_id in after_deadline_nodes:
Trust.PAYMENT.decrease(node_id)
@staticmethod
def save_task_preset(preset_name, task_type, data):
taskpreset.save_task_preset(preset_name, task_type, data)
@staticmethod
def get_task_presets(task_type):
log.info("Loading presets for {}".format(task_type))
return taskpreset.get_task_presets(task_type)
@staticmethod
def delete_task_preset(task_type, preset_name):
taskpreset.delete_task_preset(task_type, preset_name)
def _publish(self, event_name, *args, **kwargs):
if self.rpc_publisher:
self.rpc_publisher.publish(event_name, *args, **kwargs)
def lock_config(self, on=True):
self._publish(UI.evt_lock_config, on)
def config_changed(self):
self._publish(Environment.evt_opts_changed)
def __get_nodemetadatamodel(self):
return NodeMetadataModel(self.get_client_id(), self.session_id,
sys.platform, APP_VERSION,
self.get_description(), self.config_desc)
def __try_to_change_to_number(self,
old_value,
new_value,
to_int=False,
to_float=False,
name="Config"):
try:
if to_int:
new_value = int(new_value)
elif to_float:
new_value = float(new_value)
except ValueError:
log.warning("%s value '%s' is not a number", name, new_value)
new_value = old_value
return new_value
def __do_work(self):
if not self.p2pservice:
return
if self.config_desc.send_pings:
self.p2pservice.ping_peers(self.config_desc.pings_interval)
try:
self.p2pservice.sync_network()
except Exception:
log.exception("p2pservice.sync_network failed")
try:
self.task_server.sync_network()
except Exception:
log.exception("task_server.sync_network failed")
try:
self.resource_server.sync_network()
except Exception:
log.exception("resource_server.sync_network failed")
try:
self.ranking.sync_network()
except Exception:
log.exception("ranking.sync_network failed")
try:
self.check_payments()
except Exception:
log.exception("check_payments failed")
@inlineCallbacks
def __publish_events(self):
now = time.time()
delta = now - self.last_nss_time
if delta > max(self.config_desc.node_snapshot_interval, 1):
dispatcher.send(
signal='golem.monitor',
event='stats_snapshot',
known_tasks=self.get_task_count(),
supported_tasks=self.get_supported_task_count(),
stats=self.task_server.task_computer.stats,
)
dispatcher.send(
signal='golem.monitor',
event='task_computer_snapshot',
task_computer=self.task_server.task_computer,
)
# with self.snapshot_lock:
# self.__make_node_state_snapshot()
# self.manager_server.sendStateMessage(self.last_node_state_snapshot)
self.last_nss_time = time.time()
delta = now - self.last_net_check_time
if delta >= self.config_desc.network_check_interval:
self.last_net_check_time = time.time()
self._publish(Network.evt_connection, self.connection_status())
if now - self.last_tasks_time >= PUBLISH_TASKS_INTERVAL:
self._publish(Task.evt_task_list, self.get_tasks())
if now - self.last_balance_time >= PUBLISH_BALANCE_INTERVAL:
try:
gnt, av_gnt, eth = yield self.get_balance()
except Exception as exc:
log.debug('Error retrieving balance: {}'.format(exc))
else:
self._publish(
Payments.evt_balance, {
u'GNT': unicode(gnt),
u'GNT_available': unicode(av_gnt),
u'ETH': unicode(eth)
})
def __make_node_state_snapshot(self, is_running=True):
peers_num = len(self.p2pservice.peers)
last_network_messages = self.p2pservice.get_last_messages()
if self.task_server:
tasks_num = len(self.task_server.task_keeper.task_headers)
r_tasks_progs = self.task_server.task_computer.get_progresses()
l_tasks_progs = self.task_server.task_manager.get_progresses()
last_task_messages = self.task_server.get_last_messages()
self.last_node_state_snapshot = NodeStateSnapshot(
is_running, self.config_desc.node_name, peers_num, tasks_num,
self.p2pservice.node.pub_addr, self.p2pservice.node.pub_port,
last_network_messages, last_task_messages, r_tasks_progs,
l_tasks_progs)
else:
self.last_node_state_snapshot = NodeStateSnapshot(
self.config_desc.node_name, peers_num)
if self.nodes_manager_client:
self.nodes_manager_client.send_client_state_snapshot(
self.last_node_state_snapshot)
def connection_status(self):
listen_port = self.get_p2p_port()
task_server_port = self.get_task_server_port()
if listen_port == 0 or task_server_port == 0:
return u"Application not listening, check config file."
messages = []
if self.node.port_status:
statuses = self.node.port_status.split('\n')
failures = filter(lambda e: e.find('open') == -1, statuses)
messages.append(u"Port " + u", ".join(failures) + u".")
if self.get_connected_peers():
messages.append(u"Connected")
else:
messages.append(u"Not connected to Golem Network, "
u"check seed parameters.")
return u' '.join(messages)
def get_metadata(self):
metadata = dict()
# if self.ipfs_manager:
# metadata.update(self.ipfs_manager.get_metadata())
return metadata
def interpret_metadata(self, metadata, address, port, node_info):
pass
# if self.config_desc and node_info and metadata:
# seed_addresses = self.p2pservice.get_seeds()
# node_addresses = [
# (address, port),
# (node_info.pub_addr, node_info.pub_port)
# ]
# self.ipfs_manager.interpret_metadata(metadata,
# seed_addresses,
# node_addresses)
def get_status(self):
progress = self.task_server.task_computer.get_progresses()
if len(progress) > 0:
msg = u"Computing {} subtask(s):".format(len(progress))
for k, v in progress.iteritems():
msg = u"{} \n {} ({}%)\n".format(msg, k,
v.get_progress() * 100)
elif self.config_desc.accept_tasks:
msg = u"Waiting for tasks...\n"
else:
msg = u"Not accepting tasks\n"
peers = self.p2pservice.get_peers()
msg += u"Active peers in network: {}\n".format(len(peers))
return msg
def activate_hw_preset(self, name, run_benchmarks=False):
HardwarePresets.update_config(name, self.config_desc)
if hasattr(self, 'task_server') and self.task_server:
self.task_server.change_config(self.config_desc,
run_benchmarks=run_benchmarks)
def __lock_datadir(self):
if not path.exists(self.datadir):
# Create datadir if not exists yet.
# TODO: It looks we have the same code in many places
makedirs(self.datadir)
self.__datadir_lock = open(path.join(self.datadir, "LOCK"), 'w')
flags = filelock.LOCK_EX | filelock.LOCK_NB
try:
filelock.lock(self.__datadir_lock, flags)
except IOError:
raise IOError("Data dir {} used by other Golem instance".format(
self.datadir))
def _unlock_datadir(self):
# solves locking issues on OS X
try:
filelock.unlock(self.__datadir_lock)
except Exception:
pass
self.__datadir_lock.close()
def test_load_config_manager_empty(self):
mgr = EnvironmentsManager()
mgr.load_config(self.path)
assert mgr.env_config
def test_load_config_manager(self):
mgr = EnvironmentsManager()
env = Environment()
mgr.environments[env.get_id()] = env
mgr.load_config(self.path)
assert mgr.env_config
def test_get_unsupport_reasons(self):
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10)
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
# Supported task
thd = get_task_header("good")
tk.add_task_header(thd)
# Wrong version
thd = get_task_header("wrong version")
thd.fixed_header.min_version = "42.0.17"
tk.add_task_header(thd)
# Wrong environment
thd = get_task_header("wrong env")
thd.fixed_header.environment = "UNKNOWN"
tk.add_task_header(thd)
# Wrong price
thd = get_task_header("wrong price")
thd.fixed_header.max_price = 1
tk.add_task_header(thd)
# Wrong price and version
thd = get_task_header("wrong price and version")
thd.fixed_header.min_version = "42.0.17"
thd.fixed_header.max_price = 1
tk.add_task_header(thd)
# And one more with wrong version
thd = get_task_header("wrong version 2")
thd.fixed_header.min_version = "42.0.44"
tk.add_task_header(thd)
reasons = tk.get_unsupport_reasons()
# 3 tasks with wrong version
self.assertIn(
{
'avg': golem.__version__,
'reason': 'app_version',
'ntasks': 3
}, reasons)
# 2 tasks with wrong price
self.assertIn({'avg': 7, 'reason': 'max_price', 'ntasks': 2}, reasons)
# 1 task with wrong environment
self.assertIn(
{
'avg': None,
'reason': 'environment_missing',
'ntasks': 1
}, reasons)
self.assertIn(
{
'avg': None,
'reason': 'environment_not_accepting_tasks',
'ntasks': 1
}, reasons)
def test_init(self):
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10.0)
self.assertIsInstance(tk, TaskHeaderKeeper)
def test_init(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10.0)
self.assertIsInstance(tk, TaskHeaderKeeper)
