def __init__(self, service):
super(ServiceThread, self).__init__()
self.service = service
self.log = log_register('service_thread')
self._started = False
import mock
from chroma_core.models import Volume, VolumeNode, ManagedHost, LNetConfiguration
from chroma_core.models import NetworkInterface, Nid, ServerProfile
from chroma_core.models import NTPConfiguration
from chroma_core.models import CorosyncConfiguration
from chroma_core.models import PacemakerConfiguration
from chroma_core.models import Command
from chroma_core.lib.cache import ObjectCache
from chroma_core.models import StorageResourceRecord
from chroma_core.services.log import log_register
from tests.unit.chroma_core.helpers import random_str
log = log_register("synthetic_objects")
def synthetic_volume(serial=None, with_storage=True, usable_for_lustre=True):
"""
Create a Volume and an underlying StorageResourceRecord
"""
volume = Volume.objects.create()
if not serial:
serial = "foobar%d" % volume.id
attrs = {"serial": serial, "size": 8192000}
if with_storage:
from chroma_core.lib.storage_plugin.manager import storage_plugin_manager
resource_class, resource_class_id = storage_plugin_manager.get_plugin_resource_class(
import mock
from chroma_core.models import Volume, VolumeNode, ManagedHost, LNetConfiguration
from chroma_core.models import NetworkInterface, Nid, ServerProfile
from chroma_core.models import RSyslogConfiguration
from chroma_core.models import NTPConfiguration
from chroma_core.models import CorosyncConfiguration
from chroma_core.models import PacemakerConfiguration
from chroma_core.models import Command
from chroma_core.lib.cache import ObjectCache
from chroma_core.models import StorageResourceRecord
from chroma_core.services.log import log_register
from tests.unit.chroma_core.helpers import random_str
log = log_register('synthetic_objects')
def synthetic_volume(serial=None, with_storage=True, usable_for_lustre=True):
"""
Create a Volume and an underlying StorageResourceRecord
"""
volume = Volume.objects.create()
if not serial:
serial = "foobar%d" % volume.id
attrs = {'serial': serial, 'size': 8192000}
if with_storage:
from chroma_core.lib.storage_plugin.manager import storage_plugin_manager
resource_class, resource_class_id = storage_plugin_manager.get_plugin_resource_class(
import json
from chroma_core.models import ManagedHost
from chroma_core.models import ManagedTarget
from chroma_api.authentication import CsrfAuthentication
from chroma_core.services.job_scheduler import job_scheduler_notify
from chroma_core.services.job_scheduler.disabled_connection import DISABLED_CONNECTION
from iml_common.lib.agent_rpc import agent_result, agent_result_ok
from synthentic_objects import synthetic_lnet_configuration
from synthentic_objects import create_synthetic_device_info
from chroma_core.services.log import log_register
from tests.unit.chroma_core.helpers.test_api_client import TestApiClient
from tests.unit.chroma_core.helpers import helper
from iml_common.lib.date_time import IMLDateTime
log = log_register("mock_agent_rpc")
class MockAgentRpc(object):
mock_servers = {}
calls = []
host_calls = defaultdict(list)
@classmethod
def start(cls):
pass
@classmethod
def shutdown(cls):
pass
# Copyright (c) 2017 Intel Corporation. All rights reserved.
# Use of this source code is governed by a MIT-style
# license that can be found in the LICENSE file.
import sys
import traceback
import threading
from chroma_core.services.http_agent import HttpAgentRpc
from chroma_core.services.queue import AgentRxQueue
from django.db import transaction
from chroma_core.services.log import log_register
from chroma_core.models import StorageResourceRecord, ManagedHost
from chroma_core.lib.storage_plugin.query import ResourceQuery
log = log_register(__name__.split('.')[-1])
class Session(object):
def __init__(self, id, instance):
self.plugin = instance
self.id = id
self.seq = 0
class AgentPluginHandler(object):
"""Handle messages sent from the agent.
Similar to ScanDaemon, the main difference is that ScanDaemon polls plugin callbacks,
whereas AgentDaemon is driven by a message queue.
from chroma_core.models import ManagedHost
from chroma_core.services.log import log_register
from tests.unit.chroma_core.helpers import MockAgentRpc
log = log_register("mock_agent_ssh")
class MockAgentSsh(object):
ssh_should_fail = False
def __init__(self, address, log=None, console_callback=None, timeout=None):
self.address = address
def construct_ssh_auth_args(self, root_pw, pkey, pkey_pw):
return {}
def invoke(self, cmd, args={}, auth_args=None):
host = ManagedHost(address=self.address)
return MockAgentRpc._call(host, cmd, args)
def ssh(self, cmd, auth_args=None):
if self.ssh_should_fail:
from paramiko import SSHException
raise SSHException("synthetic failure")
result = self.invoke(cmd, auth_args)
if isinstance(result, int):
return (result, "", "")
else:
return (0, result, "")
import json
from chroma_core.models import ManagedHost
from chroma_core.models import ManagedTarget
from chroma_api.authentication import CsrfAuthentication
from chroma_core.services.job_scheduler import job_scheduler_notify
from chroma_core.services.job_scheduler.disabled_connection import DISABLED_CONNECTION
from chroma_core.chroma_common.lib.agent_rpc import agent_result, agent_result_ok
from synthentic_objects import synthetic_lnet_configuration
from synthentic_objects import create_synthetic_device_info
from chroma_core.services.log import log_register
from tests.unit.chroma_core.helpers.test_api_client import TestApiClient
from tests.unit.chroma_core.helpers import helper
from tests.chroma_common.lib.date_time import IMLDateTime
log = log_register('mock_agent_rpc')
class MockAgentRpc(object):
mock_servers = {}
calls = []
host_calls = defaultdict(list)
@classmethod
def start(cls):
pass
@classmethod
def shutdown(cls):
pass
def _execute_inner(self, *args, **options):
if options['verbose']:
log_enable_stdout()
if options['gevent']:
from gevent.monkey import patch_all
patch_all(thread=True)
# Gevent's implementation of select removes 'poll'
import subprocess
subprocess._has_poll = False
import django.db
django.db.connections._connections = threading.local()
if options['trace']:
class Trace(object):
def __init__(self):
self.tracefile = open('trace.log', 'w', buffering=0)
self.tracefile.write("Started at %s: %s %s\n" %
(IMLDateTime.utcnow(), args, options))
def __call__(self, frame, event, arg):
if event == "line":
try:
pyfile = frame.f_globals['__file__'].strip('co')
line = linecache.getline(pyfile, frame.f_lineno)
except KeyError:
pass
else:
if line is not None:
self.tracefile.write(
"%s:%s %s" %
(pyfile, frame.f_lineno, line))
return self
chroma_core.services.log.trace = Trace()
sys.settrace(chroma_core.services.log.trace)
from chroma_core.lib.service_config import ServiceConfig
if not ServiceConfig().configured():
# FIXME: return an error code which will prevent supervisord from restarting this service
# (using the 'exitcodes' option for the programs in the supervisord conf)
sys.stderr.write(
"Chroma is not configured, please run chroma-config setup first\n"
)
sys.exit(-1)
if not options['lightweight_rpc']:
RpcClientFactory.initialize_threads()
log_set_filename("%s.log" % options['name'])
# Respond to Ctrl+C
stopped = threading.Event()
# Ensure that threads are .start()ed before we possibly try to .join() them
setup_complete = threading.Event()
def signal_handler(*args, **kwargs):
"""Params undefined because gevent vs. threading pass
different things to handler
"""
if not setup_complete.is_set():
log.warning("Terminated during setup, exiting hard")
os._exit(0)
if not options['lightweight_rpc']:
RpcClientFactory.shutdown_threads()
for service_thread in service_mains:
log.info("Stopping %s" % service_thread.service.name)
service_thread.service.stop()
for service_thread in service_mains:
log.info("Joining %s" % service_thread.service.name)
service_thread.join()
stopped.set()
if options['gevent']:
import gevent
gevent.signal(signal.SIGINT, signal_handler)
gevent.signal(signal.SIGTERM, signal_handler)
else:
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
service_mains = []
for service_name in args:
module_path = "chroma_core.services.%s" % service_name
# Load the module
mod = __import__(module_path)
components = module_path.split('.')
for comp in components[1:]:
mod = getattr(mod, comp)
service = getattr(mod, 'Service')()
service.log = log_register(service.name)
service_thread = ServiceThread(service)
service_thread.start()
service_mains.append(service_thread)
setup_complete.set()
while not stopped.is_set():
# Using a timeout changes the behaviour of CPython's waiting so that it will
# receive signals (like ctrl-c SIGINT) immediately -- logically we don't want
# any timeout here, but a pure wait() breaks ctrl-c.
stopped.wait(10)
if len(threading.enumerate()) > 1 and not options['gevent']:
log.error("Rogue thread still running, exiting hard")
log.error([t.name for t in threading.enumerate()])
os._exit(-1)
RESPONSE_TIMEOUT = 300
"""
Max number of lightweight RPCs that can be in flight concurrently. This must
be well within the rabbitmq server's connection limit
"""
LIGHTWEIGHT_CONNECTIONS_LIMIT = 10
RESPONSE_CONN_LIMIT = 10
tx_connections = None
rx_connections = None
lw_connections = None
log = log_register("rpc")
class RpcError(Exception):
def __init__(self, description, exception_type, **kwargs):
super(Exception, self).__init__(description)
self.description = description
self.remote_exception_type = exception_type
self.traceback = kwargs.get("traceback")
def __str__(self):
return "RpcError: %s" % self.description
class RpcTimeout(Exception):
pass
import subprocess
import tempfile
import uuid
import mock
from chroma_core.models import LogMessage
from chroma_core.models import Bundle, ServerProfile
from chroma_core.lib.cache import ObjectCache
from chroma_core.services.job_scheduler.job_scheduler import JobScheduler
from chroma_core.services.log import log_register
from iml_common.lib.date_time import IMLDateTime
log = log_register('test_helper')
def random_str(length=10, prefix='', postfix=''):
test_string = (str(uuid.uuid4()).translate(None, '-'))[:length]
return "%s%s%s" % (prefix, test_string, postfix)
def synchronous_run_job(job):
for step_klass, args in job.get_steps():
step_klass(job, args, lambda x: None, lambda x: None,
mock.Mock()).run(args)
def _passthrough_create_targets(target_data):
ObjectCache.clear()
return JobScheduler().create_targets(target_data)
"""
Max number of lightweight RPCs that can be in flight concurrently. This must
be well within the rabbitmq server's connection limit
"""
LIGHTWEIGHT_CONNECTIONS_LIMIT = 10
RESPONSE_CONN_LIMIT = 10
tx_connections = None
rx_connections = None
lw_connections = None
log = log_register('rpc')
class RpcError(Exception):
def __init__(self, description, exception_type, **kwargs):
super(Exception, self).__init__(description)
self.description = description
self.remote_exception_type = exception_type
self.traceback = kwargs.get('traceback')
def __str__(self):
return "RpcError: %s" % self.description
class RpcTimeout(Exception):
pass
# Copyright (c) 2018 DDN. All rights reserved.
# Use of this source code is governed by a MIT-style
# license that can be found in the LICENSE file.
"""Chroma services may subscribe to named queues using this module. The `ServiceQueue` class is a wrapper
around an AMQP queue."""
import threading
from chroma_core.services import _amqp_connection
from chroma_core.services.log import log_register
log = log_register('queue')
class ServiceQueue(object):
"""Simple FIFO queue, multiple senders, single receiver. Payloads
must be JSON-serializable.
Subclass this for each named queue, setting the `name` class attribute.
Example declaring a queue:
::
class AcmeQueue(ServiceQueue):
name = 'acme'
Example sending to a queue:
::
AcmeQueue().put({'foo': 'bar'})
# Copyright (c) 2020 DDN. All rights reserved.
# Use of this source code is governed by a MIT-style
# license that can be found in the LICENSE file.
"""Chroma services may subscribe to named queues using this module. The `ServiceQueue` class is a wrapper
around an AMQP queue."""
import threading
from chroma_core.services import _amqp_connection
from chroma_core.services.log import log_register
log = log_register("queue")
class ServiceQueue(object):
"""Simple FIFO queue, multiple senders, single receiver. Payloads
must be JSON-serializable.
Subclass this for each named queue, setting the `name` class attribute.
Example declaring a queue:
::
class AcmeQueue(ServiceQueue):
name = 'acme'
Example sending to a queue:
::
AcmeQueue().put({'foo': 'bar'})
import mock
from tests.unit.chroma_core import helpers
from chroma_core.lib.job import Step
from chroma_core.services.log import log_register
from tests.unit.lib.iml_unit_test_case import IMLUnitTestCase
from iml_common.lib.agent_rpc import agent_result, agent_result_ok, agent_error
log = log_register('iml_test_case')
class TestJobs(IMLUnitTestCase):
def __init__(self, methodName='runTest'):
super(TestJobs, self).__init__(methodName=methodName)
self.hosts = []
self._missing_invoke_err_msg = 'Invoke attempted was unknown to InvokeCaptureTestCase, did you intend this?'
self.mock_job = mock.MagicMock()
self._invokes = []
def setUp(self):
super(TestJobs, self).setUp()
helpers.load_default_profile()
for host_id in range(0, 10):
host = helpers.synthetic_host(address='myserver%s' % host_id)
self.hosts.append(host)
self.reset_invoke_capture()
import subprocess
import tempfile
import uuid
import mock
from chroma_core.models import LogMessage
from chroma_core.models import Bundle, ServerProfile
from chroma_core.lib.cache import ObjectCache
from chroma_core.services.job_scheduler.job_scheduler import JobScheduler
from chroma_core.services.log import log_register
from iml_common.lib.date_time import IMLDateTime
log = log_register("test_helper")
def random_str(length=10, prefix="", postfix=""):
test_string = (str(uuid.uuid4()).translate(None, "-"))[:length]
return "%s%s%s" % (prefix, test_string, postfix)
def synchronous_run_job(job):
for step_klass, args in job.get_steps():
step_klass(job, args, lambda x: None, lambda x: None,
mock.Mock()).run(args)
def _passthrough_create_targets(target_data):
ObjectCache.clear()
return JobScheduler().create_targets(target_data)
import mock
from tests.unit.chroma_core import helpers
from chroma_core.lib.job import Step
from chroma_core.services.log import log_register
from tests.unit.lib.iml_unit_test_case import IMLUnitTestCase
from iml_common.lib.agent_rpc import agent_result, agent_result_ok, agent_error
log = log_register("iml_test_case")
class TestJobs(IMLUnitTestCase):
def __init__(self, methodName="runTest"):
super(TestJobs, self).__init__(methodName=methodName)
self.hosts = []
self._missing_invoke_err_msg = "Invoke attempted was unknown to InvokeCaptureTestCase, did you intend this?"
self.mock_job = mock.MagicMock()
self._invokes = []
def setUp(self):
super(TestJobs, self).setUp()
helpers.load_default_profile()
for host_id in range(0, 10):
host = helpers.synthetic_host(address="myserver%s" % host_id)
self.hosts.append(host)
self.reset_invoke_capture()
# Copyright (c) 2018 DDN. All rights reserved.
# Use of this source code is governed by a MIT-style
# license that can be found in the LICENSE file.
import sys
import traceback
import threading
from chroma_core.services.http_agent import HttpAgentRpc
from chroma_core.services.queue import AgentRxQueue
from django.db import transaction
from chroma_core.services.log import log_register
from chroma_core.models import StorageResourceRecord, ManagedHost
from chroma_core.lib.storage_plugin.query import ResourceQuery
log = log_register(__name__.split(".")[-1])
class Session(object):
def __init__(self, id, instance):
self.plugin = instance
self.id = id
self.seq = 0
class AgentPluginHandler(object):
"""Handle messages sent from the agent.
Similar to ScanDaemon, the main difference is that ScanDaemon polls plugin callbacks,
whereas AgentDaemon is driven by a message queue.
from chroma_core.models import ManagedHost
from chroma_core.services.log import log_register
from tests.unit.chroma_core.helpers import MockAgentRpc
log = log_register('mock_agent_ssh')
class MockAgentSsh(object):
ssh_should_fail = False
def __init__(self, address, log=None, console_callback=None, timeout=None):
self.address = address
def construct_ssh_auth_args(self, root_pw, pkey, pkey_pw):
return {}
def invoke(self, cmd, args={}, auth_args=None):
host = ManagedHost(address=self.address)
return MockAgentRpc._call(host, cmd, args)
def ssh(self, cmd, auth_args=None):
if self.ssh_should_fail:
from paramiko import SSHException
raise SSHException("synthetic failure")
result = self.invoke(cmd, auth_args)
if isinstance(result, int):
return (result, "", "")
else:
return (0, result, "")
def ssh_params(self):
