def __init__(self, adapter, job_status_update_callback,
job_delete_callback, job_accounting_update_callback):
self.__monitor = False
self.__thread_event = gevent.event.Event()
self.logger = LogManager.get_instance().get_logger(
self.__class__.__name__)
self.adapter = adapter
self.job_status_update_callback = job_status_update_callback
self.job_delete_callback = job_delete_callback
self.job_accounting_update_callback = job_accounting_update_callback
# Configuration
cm = ConfigManager.get_instance()
self.max_job_status_retrieval_count = int(
cm.get_config_option(
'JobMonitor', 'max_job_status_retrieval_count',
JobMonitor.MAX_JOB_STATUS_RETRIEVAL_ERROR_COUNT))
self.logger.debug('Using max job status retrieval count: %s' %
self.max_job_status_retrieval_count)
self.monitor_poll_period_in_seconds = float(
cm.get_config_option('JobMonitor',
'monitor_poll_period_in_seconds',
JobMonitor.MONITOR_POLL_PERIOD_IN_SECONDS))
self.logger.debug('Using monitor poll period: %s [seconds]' %
self.monitor_poll_period_in_seconds)
def __create_db_client(self):
""" Create db client. """
self.logger.debug('Creating db client')
cm = ConfigManager.get_instance()
db_client_config = cm.get_config_option('DBManager', 'db_client')
if db_client_config is None:
self.logger.error(
'db_client parameter missing from config file: %s' %
(cm.get_config_file()))
return
try:
dot_pos = db_client_config.find('.')
self.db_client_module = db_client_config[0:dot_pos]
self.db_client_constructor = db_client_config[dot_pos + 1:]
self.db_client_class = \
self.db_client_constructor.split('(')[0]
exec 'from %s import %s' % (self.db_client_module,
self.db_client_class)
cmd = 'db_client = %s' % self.db_client_constructor
self.logger.debug('Using %s' % cmd)
exec cmd
return db_client
except Exception, ex:
self.logger.warn('Could not create db client: %s' % ex)
def __init__(self, channelName, executor_runner):
MessageHandler.__init__(self, channelName)
cm = ConfigManager.get_instance()
self.urb_lib_path = cm.get_config_option('ExecutorHandler',
'urb_lib_path')
self.fetcher_path = cm.get_config_option('ExecutorHandler',
'fetcher_path')
self.command_executor_path = cm.get_config_option(
'ExecutorHandler', 'command_executor_path')
self.ld_library_path = cm.get_config_option('ExecutorHandler',
'ld_library_path')
self.logger.debug("LD_LIBRARY_PATH from config: %s" %
self.ld_library_path)
# Add platform specific path
ld_library_path_platform = os.path.join(self.ld_library_path,
distutils.util.get_platform())
self.ld_library_path += ':' + ld_library_path_platform
self.logger.debug("Adding platform specific LD_LIBRARY_PATH: %s" %
self.ld_library_path)
# Add from the environment
if "LD_LIBRARY_PATH" in os.environ:
self.logger.debug("Adding environment LD_LIBRARY_PATH: %s" %
os.environ['LD_LIBRARY_PATH'])
self.ld_library_path += ':' + os.environ['LD_LIBRARY_PATH']
self.logger.debug("LD_LIBRARY_PATH: %s" % self.ld_library_path)
self.executor_runner = executor_runner
self.executor_pids = []
self.executor_rets = []
self.shutdown = False
def test_register_framework():
print("Registering framework")
cm = ConfigManager.get_instance()
cf = ChannelFactory.get_instance()
framework_env = {
'URB_CONFIG_FILE' : cm.get_config_file(),
'URB_FRAMEWORK_ID' : '1',
'URB_FRAMEWORK_NAME':'test',
'URB_MASTER' : cf.get_message_broker_connection_url(),
}
adapter = K8SAdapter()
adapter.set_command(["/bin/sh", "-c", "env; sleep 100"])
max_tasks = 5
concurrent_tasks = 1
# specify unused parameters
kwargs = {'class': 'class',
'submit_options': 'submit_options'};
job_ids = adapter.register_framework(max_tasks, concurrent_tasks, framework_env, **kwargs)
print 'JOB IDs: ', job_ids
framework = {
'job_ids' : job_ids,
'name' : 'framework_name',
'id' : {'value' : 1},
'config' : {'mem' : 1024, 'disk' : 16384, 'cpus' : 1,
'ports' : '[(30000,31000)]',
'max_rejected_offers' : 1,
'max_tasks' : 5
}
}
FrameworkTracker.get_instance().add(1, framework)
assert job_ids != None
def __configure_handler(self, config_section, defaults, handler, level,
format_, datefmt):
""" Configure specified handler with a given defaults. """
cm = ConfigManager.get_instance()
cm.set_config_defaults(defaults)
handler_option = cm.get_config_option(config_section, 'handler',
handler)
# If handler_option is empty, handler cannot be instantiated.
_handler = None
if handler_option is not None:
# Handler argument format: my_module.MyHandler(arg1, arg2, ...)
# Module will be in lowercase letters, but the class
# should be capitalized.
handler_name = re.sub(r'\(.*', '', handler_option)
module_name = handler_name.split('.')[0]
try:
exec 'from urb.log import %s' % (module_name)
exec '_handler = %s' % (handler_option)
except IOError, ex:
_errno = ex.errno
import errno
# If the exception raised is an I/O permissions error,
# ignore it and disable this log handler. This allows
# non-root users to use the (system-wide) default log
# configuration
if _errno != errno.EACCES:
raise
_handler = None
except Exception, ex:
raise ConfigurationError(exception=ex)
def __create_logger_factory(self):
cm = ConfigManager.get_instance()
root_log_level = cm.get_config_option('LoggerLevels', 'root', 'error')
root_level_int = logging.getLevelName(root_log_level.upper())
root_logger = logging.getLogger('')
root_logger.root.setLevel(root_level_int)
root_logger.debug('Set root logger to %s' % root_level_int)
expressions = cm.get_config_option('LoggerLevels', 'expressions')
return LoggerFactory(expressions)
def __init__(self, defaults={}, skip_cmd_line_config=False):
self.server = None
self.shutdown = False
self.shutdown_event = gevent.event.Event()
# Config manager
ConfigManager.get_instance().set_config_defaults(defaults)
# Options
self.option_parser = OptionParser()
self.options = {}
self.args = []
if not skip_cmd_line_config:
self.option_parser.add_option('-d',
action="store_true",
dest='daemon_flag',
help="Run service as a daemon")
self.option_parser.add_option(
'-p',
'--pidfile',
dest='pid_file',
default=None,
help="Store PID in the specified file")
self.option_parser.add_option('-s',
'--silent',
action="store_true",
dest='silent',
help="No logging to stdout")
self.parse_options()
if self.get_daemon_flag() or self.get_silent():
# Disable screen logging.
ConfigManager.get_instance().set_console_log_level('notset')
# Logger
self.logger = LogManager.get_instance().get_logger('URBService')
# Configuration
self.configure()
# Signal handlers
if not skip_cmd_line_config:
signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGUSR1, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
def configure(self):
self.cm = ConfigManager.get_instance()
# we might want to get namespace from config or name it after framework
self.namespace = "default"
# cover test environment
if __name__ == '__main__' or \
'KUBERNETES_SERVICE_HOST' not in os.environ or \
'KUBERNETES_SERVICE_PORT' not in os.environ:
client.Configuration().host = "http://127.0.0.1:8001"
# config.load_kube_config()
else:
# to workaround for:
# SSLError hostname '10.0.0.1' doesn't match either of 'kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'
# https://github.com/kubernetes-incubator/client-python/issues/254
# has to set environment variable:
os.environ['KUBERNETES_SERVICE_HOST'] = 'kubernetes'
config.load_incluster_config()
def __create_adapter(self, interface_name, channel_name):
""" Create adapter instance. """
self.logger.debug('Creating adapter for interface: %s' % interface_name)
adapter_option_name = '%s_adapter' % interface_name
adapter_path_option_name = '%s_adapter_path' % interface_name
cm = ConfigManager.get_instance()
adapter_path = cm.get_config_option('AdapterManager', adapter_path_option_name)
adapter_config = cm.get_config_option('AdapterManager', adapter_option_name)
self.logger.debug('adapter_path=%s, adapter_config=%s' % (adapter_path, adapter_config))
if adapter_config is None:
raise ConfigurationError(
'Adapter parameter %s missing from config file: %s'
% (adapter_option_name, cm.get_config_file()))
dot_pos = adapter_config.find('.')
if dot_pos == -1:
raise ConfigurationError("Incorrect adapter format: should be in form <handlername>_adapter=adapter_module.AdapterClass(arg1, arg2, ...)")
self.adapter_module = adapter_config[:dot_pos]
self.adapter_constructor = adapter_config[dot_pos+1:]
self.logger.trace('adapter_module=%s, adapter_constructor=%s' % (self.adapter_module, self.adapter_constructor))
try:
self.adapter_class = self.adapter_constructor.split('(')[0]
self.logger.trace('adapter_class=%s' % self.adapter_class)
if adapter_path is not None:
if adapter_path[0] == ".":
self.logger.trace('adapter_path is relative')
# relative path
adapter_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), adapter_path)
self.logger.trace('adapter_path=%s' % adapter_path)
sys.path.append(adapter_path)
self.logger.trace('Executing: from %s import %s' % (self.adapter_module, self.adapter_class))
exec 'from %s import %s' % (self.adapter_module, self.adapter_class)
exec 'adapter = %s' % self.adapter_constructor
else:
adapter_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../..", self.adapter_module)
self.logger.trace('adapter_path=%s' % adapter_path)
sys.path.append(adapter_path)
self.logger.trace('Executing: from %s.%s import %s' % (self.adapter_module, self.adapter_module, self.adapter_class))
exec 'from %s.%s import %s' % (self.adapter_module, self.adapter_module, self.adapter_class)
exec 'adapter = %s' % self.adapter_constructor
except Exception, ex:
self.logger.error('AdapterManager configuration error: %s' % ex)
raise ConfigurationError(exception=ex)
def configure(self):
# all jobs inherit URB master namespace passed via environment variable
self.namespace = os.environ.get('URB_NAMESPACE', 'default')
self.logger.info("K8s namespace: %s" % self.namespace)
self.cm = ConfigManager.get_instance()
# cover test environment
if __name__ == '__main__' or \
'KUBERNETES_SERVICE_HOST' not in os.environ or \
'KUBERNETES_SERVICE_PORT' not in os.environ:
self.logger.info("In test environment")
# client.Configuration().host = "http://127.0.0.1:8001"
config.load_kube_config()
else:
# to workaround for:
# SSLError hostname '10.0.0.1' doesn't match either of 'kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'
# https://github.com/kubernetes-incubator/client-python/issues/254
# has to set environment variable:
os.environ['KUBERNETES_SERVICE_HOST'] = 'kubernetes.default'
config.load_incluster_config()
def __init__(self):
""" Initialize factory instance. """
# Only initialize once.
if ChannelFactory.__instance is not None:
return
cm = ConfigManager.get_instance()
message_broker = cm.get_config_option('ChannelFactory',
'message_broker')
self.logger = LogManager.get_instance().get_logger(
self.__class__.__name__)
self.logger.debug('Using message broker: %s' % message_broker)
if message_broker is None:
raise ConfigurationError(
'Message broker parameter missing from config file: %s' %
cm.get_config_file())
dot_pos = message_broker.find('.')
self.message_broker_module = message_broker[0:dot_pos]
self.message_broker_constructor = message_broker[dot_pos + 1:]
self.message_broker_class = \
self.message_broker_constructor.split('(')[0]
self.message_broker = self.__get_message_broker()
def configure(self):
cm = ConfigManager.get_instance()
config_items = cm.get_config_items('Service')
self.handler_list = []
try:
for (key, value) in config_items:
if key.endswith('_handler') and value is not None:
handler_module = value.split('.')[0]
handler_constructor = '.'.join(value.split('.')[1:])
handler_class = handler_constructor.split('(')[0]
self.logger.info(
'Instantiating handler: %s, handler_module=%s ,handler_class=%s'
% (handler_constructor, handler_module, handler_class))
exec 'from %s import %s' % (handler_module, handler_class)
exec 'handler = %s' % (handler_constructor)
handler.register_shutdown_callback(self.shutdown_callback)
self.handler_list.append(handler)
except Exception, ex:
self.logger.error('URBService configuration error: %s' % ex)
raise ConfigurationError(exception=ex)
def configure(self):
cm = ConfigManager.get_instance()
self.service_monitor_endpoint = cm.get_config_option('ExecutorRunner',
'service_monitor_endpoint')
mesos_master_endpoint = cm.get_config_option('ExecutorRunner',
'mesos_master_endpoint')
self.mesos_work_dir = cm.get_config_option('ExecutorRunner',
'mesos_work_dir') % { 'tmp' : os.environ.get('TMP','') }
# Create our working directory before we start logging
if not os.path.exists(self.mesos_work_dir):
os.makedirs(self.mesos_work_dir)
os.chdir(self.mesos_work_dir)
self.logger = LogManager.get_instance().get_logger(self.__class__.__name__)
self.logger.debug('Config file: %s' % cm.get_config_file())
cf = ChannelFactory.get_instance()
self.service_monitor_channel = cf.create_channel(self.service_monitor_endpoint)
self.mesos_channel = cf.create_channel(mesos_master_endpoint)
self.channel_name = cf.get_unique_channel_name()
self.logger.debug('Got my channel name: %s' % self.channel_name)
self.host = cf.get_message_broker_connection_host()
self.port = cf.get_message_broker_connection_port()
# The service will send messages to our notify channel
channel_id = MessagingUtility.get_endpoint_id(self.channel_name)
self.notify_channel_name = MessagingUtility.get_notify_channel_name(None,channel_id)
self.handler_list = [ExecutorHandler(self.notify_channel_name,self)]
# Get various id objects
self.framework_id = {
'value' : os.environ['URB_FRAMEWORK_ID']
}
if 'JOB_ID' in os.environ:
self.job_id = os.environ['JOB_ID']
self.task_id = os.environ.get('SGE_TASK_ID', '1')
slave_id = NamingUtility.create_slave_id(self.job_id, self.task_id, self.notify_channel_name)
else:
self.job_id = uuid.uuid1().hex
self.logger.error('Environment variable JOB_ID is not defined, autogenerating job id: ' % self.job_id)
self.task_id = "1"
slave_id = NamingUtility.create_slave_id(self.job_id, self.task_id , self.notify_channel_name)
self.slave_id = {
"value" : slave_id
}
self.logger.debug('slave id: %s' % self.slave_id)
self.slave_info = {
'hostname' : self.host,
'port' : self.port,
'id' : self.slave_id
}
self.dummy_framework_info = {
'name':'default',
'user':'******',
}
self.heartbeat_channel_info = {
'channel_id' : self.notify_channel_name,
'framework_id' : self.framework_id['value'],
'slave_id' : self.slave_id['value'],
'endpoint_type' : 'executor_runner',
}
def __configure_handlers(self):
""" Configure all log handlers from the config file. """
cm = ConfigManager.get_instance()
# Config manager will return the requested setting, or
# its own default values if the default value is not passed in.
handler = 'stream_log_handler.StreamLogHandler(sys.stderr,)'
level = cm.get_console_log_level(None)
format_ = cm.get_log_record_format(None)
datefmt = cm.get_log_date_format(None)
defaults = {
'level': cm.get_console_log_level(),
'format': cm.get_log_record_format(),
'datefmt': cm.get_log_date_format(),
'handler': handler
}
console_handler = None
if cm.get_console_log_level() != 'notset':
console_handler = self.__configure_handler('ConsoleLogging',
defaults, handler,
level, format_, datefmt)
if console_handler is not None:
self.user_handler_list.append(console_handler)
# Local file logging (only turned on if it is in the config file).
handler = None
level = cm.get_file_log_level(None)
defaults['level'] = cm.get_file_log_level()
defaults['handler'] = handler
# Parse all of the file loggers present in the config file
config_sections = cm.get_config_sections()
for config_section in config_sections:
if config_section.startswith('FileLogging'):
fileHandler = self.__configure_handler(config_section,
defaults, handler,
level, format_, datefmt)
if fileHandler is not None:
self.system_handler_list.append(fileHandler)
# Remote logging (only turned on if it is in the config file).
remoteHandler = self.__configure_handler('RemoteLogging', defaults,
handler, level, format_,
datefmt)
if remoteHandler is not None:
self.system_handler_list.append(remoteHandler)
# Add handlers to the root logger. Use logging class here
# to make sure we can have a logger when we parse the
# logger expressions
root_logger = logging.getLogger('')
for handler in self.user_handler_list + self.system_handler_list:
root_logger.addHandler(handler)
# Now get a logger factory based on our current config
self.logger_factory = self.__create_logger_factory()
def __init__(self, defaults={}):
self.done = False
# Config manager
ConfigManager.get_instance().set_config_defaults(defaults)
self.configure()
def create_config_manager():
from urb.config.config_manager import ConfigManager
cm = ConfigManager.get_instance()
print "set log file to %s" % LOG_FILE
cm.set_log_file(LOG_FILE)
cm.set_file_log_level('trace')
def configure(self):
self.logger.debug('configure')
cm = ConfigManager.get_instance()
self.executor_runner_path = cm.get_config_option('ExecutorHandler', 'executor_runner_path', 'executor-runner')
from common_utils import create_service_channel
from common_utils import need_python_27
from urb.config.config_manager import ConfigManager
from mesoshttp.client import MesosClient
if os.environ.get("URB_CONFIG_FILE") is None:
raise Exception("URB_CONFIG_FILE environment variable has to be defined")
from gevent import monkey
monkey.patch_all()
SERVICE_THREAD = create_service_thread(serve_time=100)
CM = ConfigManager.get_instance()
HTTP_PORT = CM.get_config_option("Http", "scheduler_port", "5060")
URL = "http://127.0.0.1:" + HTTP_PORT
class Test(object):
class MesosFramework(threading.Thread):
def __init__(self, client):
threading.Thread.__init__(self)
self.client = client
self.exited = False
def run(self):
try:
self.client.register()
except KeyboardInterrupt:
print('Stop requested by user, stopping framework....')