def test_config_logging(self):
"""Tests wether logging is properly configured"""
datadir = os.path.join(self.path, "data_test")
logsdir = os.path.join(datadir, "logs")
suffix = "_tests"
with patch('logging.config.dictConfig') as m_dconfig:
config_logging(suffix, datadir=datadir)
m_dconfig.assert_called_once_with(ANY)
self.assertTrue(os.path.exists(logsdir))
def run_requesting_node(datadir, num_subtasks=3):
client = None
def shutdown():
client and client.quit()
reactor.running and reactor.callFromThread(reactor.stop)
logging.shutdown()
if os.path.exists(datadir):
_print_golem_log(datadir)
shutil.rmtree(datadir)
atexit.register(shutdown)
global node_kind
node_kind = "REQUESTOR"
start_time = time.time()
report("Starting in {}".format(datadir))
from golem.core.common import config_logging
config_logging(datadir=datadir, loglevel="DEBUG")
client = create_client(datadir)
client.are_terms_accepted = lambda: True
client.start()
report("Started in {:.1f} s".format(time.time() - start_time))
dummy_env = DummyEnvironment()
client.environments_manager.add_environment(dummy_env)
params = DummyTaskParameters(1024, 2048, 256, 0x0001ffff)
task = DummyTask(client.get_node_name(), params, num_subtasks,
client.keys_auth.public_key)
task.initialize(DirManager(datadir))
task_rpc.enqueue_new_task(client, task)
port = client.p2pservice.cur_port
requestor_addr = "{}:{}".format(client.node.prv_addr, port)
report("Listening on {}".format(requestor_addr))
def report_status():
while True:
time.sleep(1)
if not task.finished_computation():
continue
if task.verify_task():
report("Task finished")
else:
report("Task failed")
shutdown()
return
reactor.callInThread(report_status)
reactor.run()
return client # Used in tests, with mocked reactor
def run_computing_node(datadir, peer_address, fail_after=None):
client = None
def shutdown():
client and client.quit()
reactor.running and reactor.callFromThread(reactor.stop)
logging.shutdown()
if os.path.exists(datadir):
_print_golem_log(datadir)
shutil.rmtree(datadir)
atexit.register(shutdown)
global node_kind
node_kind = "COMPUTER "
start_time = time.time()
report("Starting in {}".format(datadir))
from golem.core.common import config_logging
config_logging(datadir=datadir, loglevel="DEBUG")
client = create_client(datadir)
client.are_terms_accepted = lambda: True
client.start()
client.task_server.task_computer.support_direct_computation = True
report("Started in {:.1f} s".format(time.time() - start_time))
dummy_env = DummyEnvironment()
dummy_env.accept_tasks = True
client.environments_manager.add_environment(dummy_env)
report("Connecting to requesting node at {}:{} ..."
.format(peer_address.address, peer_address.port))
client.connect(peer_address)
def report_status(fail_after=None):
t0 = time.time()
while True:
if fail_after and time.time() - t0 > fail_after:
report("Failure!")
reactor.callFromThread(reactor.stop)
shutdown()
return
time.sleep(1)
reactor.callInThread(report_status, fail_after)
reactor.run()
return client # Used in tests, with mocked reactor
def start():
flags = dict(
interactive=('-i', '--interactive'),
address=('-a', '--address'),
port=('-p', '--port'),
)
flag_options = dict(
interactive=dict(dest="interactive",
action="store_true",
default=False,
help="Enter interactive mode"),
address=dict(dest="address",
type=str,
default='localhost',
help="Golem node's RPC address"),
port=dict(dest="port",
type=int,
default=61000,
help="Golem node's RPC port"),
)
# process initial arguments
parser = argparse.ArgumentParser(add_help=False)
for flag_name, flag in flags.items():
parser.add_argument(*flag, **flag_options[flag_name])
args = sys.argv[1:]
parsed, forwarded = parser.parse_known_args(args)
# setup logging if in interactive mode
interactive = parsed.interactive
if interactive:
config_logging("_cli")
cli = CLI()
else:
import logging
logging.raiseExceptions = 0
cli = CLI(main_parser=parser, main_parser_options=flag_options)
# run the cli
ws_cli = WebSocketCLI(cli,
host=parsed.address,
port=parsed.port,
realm=u'golem')
ws_cli.execute(forwarded, interactive=interactive)
def test_config_logging(self):
"""Tests wether logging is properly configured"""
datadir = os.path.join(self.path, "data_test")
logsdir = os.path.join(datadir, "logs")
suffix = "_tests"
with patch('logging.config.dictConfig') as m_dconfig:
config_logging(suffix, datadir=datadir)
m_dconfig.assert_called_once_with(ANY)
# test with a level
m_dconfig.reset_mock()
t_lvl = 'WARNING'
config_logging(suffix, datadir=datadir, loglevel=t_lvl)
self.assertEqual(m_dconfig.call_args[0][0]['root']['level'], t_lvl)
self.assertTrue(os.path.exists(logsdir))
def start(gui, payments, datadir, node_address, rpc_address, peer, task, qt, version, m):
freeze_support()
if version:
from golem.core.variables import APP_VERSION
print ("GOLEM version: {}".format(APP_VERSION))
return 0
# Workarounds for pyinstaller executable
sys.modules['win32com.gen_py.os'] = None
sys.modules['win32com.gen_py.pywintypes'] = None
sys.modules['win32com.gen_py.pythoncom'] = None
config = dict(datadir=datadir, transaction_system=payments)
if rpc_address:
config['rpc_address'] = rpc_address.address
config['rpc_port'] = rpc_address.port
# Crossbar
if m == 'crossbar.worker.process':
start_crossbar_worker(m)
# Qt GUI
elif qt:
delete_reactor()
from gui.startgui import start_gui, check_rpc_address
address = '{}:{}'.format(rpc_address.address, rpc_address.port)
start_gui(check_rpc_address(ctx=None, param=None,
address=address))
# Golem
elif gui:
delete_reactor()
from gui.startapp import start_app
start_app(rendering=True, **config)
# Golem headless
else:
from golem.core.common import config_logging
config_logging(datadir=datadir)
node = OptNode(node_address=node_address, **config)
node.initialize()
node.connect_with_peers(peer)
node.add_tasks(task)
node.run(use_rpc=True)
def main(datadir, rpc_address):
from twisted.internet import reactor
if rpc_address:
host = rpc_address.address
port = rpc_address.port
else:
host = 'localhost'
port = 61000
config_logging(datadir=datadir)
logger = logging.getLogger('events')
address = WebSocketAddress(host, port, realm=u'golem')
events = build_handlers(logger)
rpc_session = EventLoggingSession(logger, address, events=events)
rpc_session.connect(auto_reconnect=True)
reactor.run()
import uuid
from os import path
import docker.errors
import requests
from mock import patch, mock
from golem.core.common import config_logging
from golem.core.common import is_windows, nt_path_to_posix_path
from golem.core.simpleenv import get_local_datadir
from golem.docker.image import DockerImage
from golem.docker.job import DockerJob, container_logger
from golem.tools.ci import ci_skip
from test_docker_image import DockerTestCase
config_logging('docker_test')
class TestDockerJob(DockerTestCase):
"""Common superclass for Docker job tests"""
def _get_test_repository(self):
"""Abstract method, should be overriden by subclasses"""
pass
def _get_test_tag(self):
return "latest"
TEST_SCRIPT = "print 'Adventure Time!'\n"
def setUp(self):
def setUpClass(cls):
config_logging()
def start():
freeze_support()
delete_reactor()
flags = dict(interactive=('-i', '--interactive'),
mainnet=('-m', '--mainnet'),
address=('-a', '--address'),
port=('-p', '--port'),
trust=('-t', '--verify-trust'),
datadir=("-d", "--datadir"))
flag_options = dict(
interactive=dict(dest="interactive",
action="store_true",
default=False,
help="Enter interactive mode"),
mainnet=dict(dest="mainnet",
action="store_true",
default=False,
help="Use mainnet chain"),
address=dict(dest="address",
type=str,
default=CROSSBAR_HOST,
help="Golem node's RPC address"),
port=dict(dest="port",
type=int,
default=CROSSBAR_PORT,
help="Golem node's RPC port"),
datadir=dict(dest="datadir",
default=None,
type=click.Path(
exists=True,
file_okay=False,
readable=True,
),
help="Golem node's data dir"),
trust=dict(dest="verify_trust",
action="store_true",
default=False,
help="Verify Golem node's certificate"),
)
# process initial arguments
parser = argparse.ArgumentParser(add_help=False)
for flag_name, flag in flags.items():
parser.add_argument(*flag, **flag_options[flag_name])
args = sys.argv[1:]
parsed, forwarded = parser.parse_known_args(args)
install_reactor()
# platform trust settings
if not parsed.verify_trust:
disable_platform_trust()
# setup logging if in interactive mode
interactive = parsed.interactive
if interactive:
config_logging("_cli")
cli = CLI()
else:
import logging
logging.raiseExceptions = 0
cli = CLI(main_parser=parser, main_parser_options=flag_options)
check_golem_running(parsed.datadir, parsed.mainnet)
if parsed.mainnet:
set_environment('mainnet', None)
datadir = get_local_datadir('default', root_dir=parsed.datadir)
working_dir = os.path.join(datadir, CROSSBAR_DIR)
# run the cli
ws_cli = WebSocketCLI(cli,
CertificateManager(working_dir),
host=parsed.address,
port=parsed.port)
ws_cli.execute(forwarded, interactive=interactive)
import logging
import click
from apps.appsmanager import AppsManager
from golem.core.common import config_logging
from golem.core.deferred import install_unhandled_error_logger
from golem.network.transport.tcpnetwork import SocketAddress
from golem.rpc.mapping.core import CORE_METHOD_MAP
from golem.rpc.session import object_method_map, Session, WebSocketAddress
from ipaddress import AddressValueError
from twisted.internet.defer import inlineCallbacks
config_logging("_gui")
logger = logging.getLogger("app")
install_unhandled_error_logger()
apps_manager = AppsManager()
apps_manager.load_apps()
def install_qt5_reactor():
import qt5reactor
qt5reactor.install()
from twisted.internet import reactor
return reactor
def register_rendering_task_types(logic):
from gui.view.widget import TaskWidget
for app in apps_manager.apps.values():
task_type = app.task_type_info(TaskWidget(app.widget), app.controller)
def start_client(start_ranking, datadir=None,
transaction_system=False, client=None,
reactor=None, **config_overrides):
config_logging("client", datadir=datadir)
logger = logging.getLogger("golem.client")
install_unhandled_error_logger()
if not reactor:
from twisted.internet import reactor
process_monitor = None
from golem.core.processmonitor import ProcessMonitor
from golem.docker.manager import DockerManager
from golem.rpc.router import CrossbarRouter
process_monitor = None
if not client:
client = Client(datadir=datadir, transaction_system=transaction_system, **config_overrides)
docker_manager = DockerManager.install(client.config_desc)
docker_manager.check_environment()
environments = load_environments()
client.sync()
for env in environments:
client.environments_manager.add_environment(env)
client.environments_manager.load_config(client.datadir)
config = client.config_desc
methods = object_method_map(client, CORE_METHOD_MAP)
router = CrossbarRouter(
host=config.rpc_address,
port=config.rpc_port,
datadir=client.datadir
)
session = Session(router.address, methods=methods)
def router_ready(*_):
session.connect().addCallbacks(session_ready, start_error)
def session_ready(*_):
global process_monitor
logger.info('Router session ready. Starting client...')
try:
client.configure_rpc(session)
logger.debug('client.start()')
client.start()
logger.debug('after client.start()')
except SystemExit:
raise
except Exception as exc:
logger.exception("Client process error: {}"
.format(exc))
logger.info('Starting GUI process...')
gui_process = start_gui(router.address)
process_monitor = ProcessMonitor(gui_process)
process_monitor.add_callbacks(stop_reactor)
logger.info('Starting process monitor...')
process_monitor.start()
reactor.addSystemEventTrigger("before", "shutdown", router.stop)
router.start(reactor, router_ready, start_error)
if start_ranking:
client.ranking.run(reactor)
try:
reactor.run()
except ReactorAlreadyRunning:
logger.debug("Client process: reactor is already running")
if process_monitor:
process_monitor.exit()
# -*- coding: utf-8 -*-
import logging
import unittest
import unittest.mock as mock
from golem.core.common import config_logging
from golem.testutils import PEP8MixIn
config_logging(suffix='_test_logging')
logger = logging.getLogger('test.logging')
class TestLogging(unittest.TestCase, PEP8MixIn):
PEP8_FILES = [
"golem/utils.py",
]
@mock.patch('logging.Handler.handleError')
def test_unicode_formatter(self, handleError_mock):
problematic_s = '\xe9\x01\x03'
msg = problematic_s + ' %s'
logger.warning(msg, problematic_s)
try:
raise ValueError(problematic_s)
except ValueError:
logger.exception('test')
for call in handleError_mock.call_args_list:
# This helps in debugging. Call list will be empty if test succeeds.
args, _kwargs = call
record = args[0]
def start(monitor, concent, datadir, node_address, rpc_address, peer, mainnet,
net, geth_address, password, accept_terms, accept_concent_terms,
accept_all_terms, version, log_level, enable_talkback, m):
# These are done locally since they rely on golem.config.active to be set
from golem.config.active import CONCENT_VARIANT
from golem.appconfig import AppConfig
from golem.node import Node
freeze_support()
delete_reactor()
# Crossbar
if m == 'crossbar.worker.process':
start_crossbar_worker(m)
return
if version:
print("GOLEM version: {}".format(golem.__version__))
return 0
generate_rpc_certificate(datadir)
# Workarounds for pyinstaller executable
sys.modules['win32com.gen_py.os'] = None
sys.modules['win32com.gen_py.pywintypes'] = None
sys.modules['win32com.gen_py.pythoncom'] = None
app_config = AppConfig.load_config(datadir)
config_desc = ClientConfigDescriptor()
config_desc.init_from_app_config(app_config)
config_desc = ConfigApprover(config_desc).approve()
if rpc_address:
config_desc.rpc_address = rpc_address.address
config_desc.rpc_port = rpc_address.port
if node_address:
config_desc.node_address = node_address
# Golem headless
install_reactor()
from golem.core.common import config_logging
config_logging(datadir=datadir,
loglevel=log_level,
config_desc=config_desc)
log_golem_version()
log_platform_info()
log_ethereum_chain()
log_concent_choice(CONCENT_VARIANT)
node = Node(
datadir=datadir,
app_config=app_config,
config_desc=config_desc,
peers=peer,
use_monitor=monitor,
use_talkback=enable_talkback,
concent_variant=CONCENT_VARIANT,
geth_address=geth_address,
password=password,
)
if accept_terms:
node.accept_terms()
if accept_concent_terms:
node.accept_concent_terms()
if accept_all_terms:
node.accept_terms()
node.accept_concent_terms()
node.start()