class trace:
'''
Provides caching method result by method input parameters.
'''
logger = logging.get_logger()
def __init__(self, *args, **options):
pass
def wrapper(self, func, *args, **kwargs):
'''
Wraps the function.
@param func: function
@return: depends on function
'''
start_time = time.time()
try:
# Returning the result
return func(*args, **kwargs)
except Exception, error:
self.logger.debug('Error on executing function[%s.%s]:%s' %
(func.__module__, func.__name__, error))
raise
finally:
class PyroListener(Listener):
'''
Pyro listener.
'''
logger = logging.get_logger(name='PYRO')
communicator_logger = logging.get_logger(name='communicator')
def __init__(self, communicator, name, params, client_request_class=None):
Listener.__init__(self,
communicator,
name,
params,
client_request_class=client_request_class)
if not params.has_key('service_name'):
params['service_name'] = self.get_name()
Pyro.config.PYRO_MULTITHREADED = 1
Pyro.core.initServer()
self._daemon = Pyro.core.Daemon(host=self.host, port=self.port)
#self.__daemon.setNewConnectionValidator(ConnectionValidator())
self.add_service(PyroDispatcher(self), name=params['service_name'])
self._exception_manager = PyroExceptionManager()
def login(self, ip, user_name, password, **options):
'''
Logins in application.
@param ip: client IP address
@param user_name: user name
@param password: user password
'''
try:
return Listener.login(self, ip, user_name, password, **options)
except DeltaException, error:
# As login method calls login_ex method of the communicator, the raised
# exception is already formatted when it reaches this method; hence, no
# reformatting is needed.
raise error
except Exception, error:
raise error
class XmlrpcListener(Listener):
logger = logging.get_logger(name='xmlrpc')
def __init__(self, communicator, name, params, client_request_class=None):
Listener.__init__(self,
communicator,
name,
params,
client_request_class=client_request_class)
if not params.has_key('service_name'):
params['service_name'] = self.get_name()
host = params.get('host')
port = int(params.get('port'))
self._server = DocXMLRPCServer(addr=(host, port),
allow_none=1,
logRequests=0)
self.add_service(XmlrpcDispatcher(self), name=params['service_name'])
def __get_instance_methods__(self, instance, prefix=None):
methods = {}
for o in dir(instance):
if not o.startswith('_'):
method = getattr(instance, o)
if method.__class__.__name__ == 'instancemethod':
if prefix:
methods["%s.%s" % (prefix, o)] = method
else:
methods[o] = method
return methods
def add_service(self, service, **kwargs):
'''
Add a service to listener.
@param service: service object
'''
name = kwargs.get('name', None)
methods = self.__get_instance_methods__(service, name)
for method_name, method in methods.items():
self._server.register_function(method, method_name)
def start(self):
self._server.serve_forever()
def stop(self, force):
try:
self._server.shutdown()
except Exception, error:
XmlrpcListener.logger.warning(error)
class TransactionCoordinatorRequestProcessorManagerHook(
RequestProcessorManagerHook):
LOGGER = logging_services.get_logger(name='request.coordinator')
def __init__(self):
RequestProcessorManagerHook.__init__(
self, 'request_processor_manager.hook.coordinator')
def on_process(self, request, **options):
'''
It will be called when a request starts to be processed
@param ClientRequest request: client request
'''
coordinator_services.record_request(request)
logging_services.debug(
'Request [{0}] for service [{1}] recorded.'.format(
request.id, request.command_key))
def on_process_completed(self, request, response, **options):
'''
It will be called when a request is completed.
@param ClientRequest request: client request
@param ServerResponse response: server response
'''
coordinator_services.set_completed(request)
logging_services.debug(
'Request [{0}] for service [{1}] completed.'.format(
request.id, request.command_key))
def on_process_failed(self, request, error, **options):
'''
It will be called when a request is failed.
@param ClientRequest request: client request
@param Exception error: error content
'''
coordinator_services.set_failed(request, error.message)
logging_services.debug(
'Request [{0}] for service [{1}] failed.'.format(
request.id, request.command_key))
class TaskExecutor:
logger = logging.get_logger(name = 'scheduler')
def __init__(self, on_running_task):
self.running_threads = {}
self.on_running_task = on_running_task
self.tasks_lock = Lock()
def __run_task__(self,task):
try:
self.tasks_lock.acquire()
if self.on_running_task:
self.on_running_task(task)
self.running_threads[task.task_id]= task
task.status = Task.RUNNING
task.run()
self.running_threads.pop(task.task_id)
except Exception, e:
TaskExecutor.logger.exception(e)
raise
finally:
class BatchProcessManager(DeltaObject):
'''
Provides management on process units.
'''
logger = logging.get_logger(name='batch')
def __init__(self):
DeltaObject.__init__(self)
self.__process_units = {}
def __get_process_unit_data__(self, name):
'''
Returns specific batch processor.
@param name: name of batch processor.
'''
try:
return self.__process_units[name]
except KeyError:
raise BatchProcessManagerException("Process unit[%s] not found." %
name)
def __get_parallel_executor__(self, process_unit_name):
'''
Returns parallel executor of a process unit.
@param process_unit_name: process unit name
@return: ParallelExecutor
'''
# Getting process unit data
data = self.__get_process_unit_data__(process_unit_name)
# data is a tuple of (process_unit, parallel_executor)
return data[1]
def get_process_units(self, status=None):
'''
Returns all process units by the given status
@param status: status
'Stopped'
'InProgress'
'Suspended'
'Completed'
'Ready'
'NotReady'
'Failed'
@return: [ProcessUnit]
'''
process_units = []
for process_unit, parallel_executor in self.__process_units.values():
#Commented because batch.list command returned Empty Result.
#if parallel_executor is not None:
if not status or parallel_executor.get_status() == status:
process_units.append(process_unit)
return process_units
def get_process_unit(self, name):
'''
Returns specific process unit.
@param name: name of process unit.
@return: ProcessUnit
'''
# Getting process unit data
data = self.__get_process_unit_data__(name)
# data is a tuple of (process_unit, parallel_executor)
return data[0]
def add_process_unit(self, process_unit, **options):
'''
Adds a process unit to batch processor.
If a process unit with the same ID exists,
it will override it.
@param process_unit: process unit instance
'''
# Getting process unit name
name = process_unit.get_name()
# Adding process unit data as a tuple
# (process unit instance, parallel executor instance)
self.__process_units[name] = (process_unit, None)
def __before_start_process_unit__(self, process_unit, parallel_executor,
thread_count, params):
# TODO: BatchProcessManager start hooks
# Calling process unit to creating it's chunks
process_unit_name = process_unit.get_name()
BatchProcessManager.logger.info(
"Process unit[%s] started it's chunks creation." %
process_unit_name)
chunks = process_unit.create_chunks()
BatchProcessManager.logger.info(
"Process unit[%s] finished it's chunks creation." %
process_unit_name)
if chunks is None or len(chunks) == 0:
message = _(
'Can not start process unit[{unit_name}] without chunks.')
process_unit.get_message_stream().error(
message.format(unit_name=process_unit_name))
raise BatchProcessManagerException(
message.format(unit_name=process_unit_name))
message = _(
'Process unit[{unit_name}] will be started by {chunk_count} chunks.'
)
BatchProcessManager.logger.info(
message.format(unit_name=process_unit_name,
chunk_count=len(chunks)))
# Adding created chunks as parallel executor parameters
parallel_executor.add_parameters(chunks)
def __after_finished_process_unit__(self, process_unit):
process_unit.done()
BatchProcessManager.logger.info('Process unit[%s] completed.' %
process_unit.get_name())
process_unit.get_message_stream().info('Process unit[%s] completed.' %
process_unit.get_name())
# TODO: BatchProcessManager complete hooks
def __on_proccess_unit_failed__(self, process_unit, error, error_trace):
try:
BatchProcessManager.logger.error(
'Process unit[%s] failed:%s.' %
(process_unit.get_name(), error_trace))
process_unit.get_message_stream().error(
'Process unit[%s] failed:%s.' %
(process_unit.get_name(), error_trace))
process_unit.failed(error)
except:
BatchProcessManager.logger.error(
'Process unit[%s] failed:%s.' %
(process_unit.get_name(), traceback.format_exc()))
process_unit.get_message_stream().error(
'Process unit[%s] failed:%s.' %
(process_unit.get_name(), traceback.format_exc()))
def __before_start_process_unit_chunk__(self, process_unit, chunk):
BatchProcessManager.logger.info(
'Process unit[%s] is going to start chunk[%s].' %
(process_unit.get_name(), chunk))
process_unit.get_message_stream().info(
'Process unit[%s] is going to start chunk[%s].' %
(process_unit.get_name(), chunk))
def __after_complete_process_unit_chunk__(self, process_unit, chunk,
result):
BatchProcessManager.logger.info(
'Process unit[%s] finished chunk[%s].' %
(process_unit.get_name(), chunk))
process_unit.get_message_stream().info(
'Process unit[%s] finished chunk[%s].' %
(process_unit.get_name(), chunk))
def __on_process_unit_chunk_failed__(self, process_unit, chunk, error):
error_detail = traceback.format_exc()
BatchProcessManager.logger.error(
'Process unit[%s] failed on chunk[%s] : %s\n Trace: %s.' %
(process_unit.get_name(), chunk, error, error_detail))
process_unit.get_message_stream().error(
'Process unit[%s] failed on chunk[%s] : %s\n Trace: %s.' %
(process_unit.get_name(), chunk, error, error_detail))
def __get_process_unit_config__(self, name):
'''
Returns process unit configuration from batch.config file.
@param name: process unit name
'''
# Getting batch configuration store
config_store = config.get_app_config_store('batch')
# Getting default parameters from configuration file.
if config_store.has_section(name):
return config_store.get_section_data(name)
return DynamicObject(thread_count=1)
def start(self, process_unit_name, **options):
'''
Starts a process unit.
@param process_unit_name: process unit name
'''
BatchProcessManager.logger.info('Loading process unit[%s]' %
process_unit_name)
process_unit = None
try:
parallel_executor = self.__get_parallel_executor__(
process_unit_name)
if parallel_executor is not None and not parallel_executor.can_run(
):
raise BatchProcessManagerException(
'Process unit[%s] is %s.' %
(process_unit_name, self.get_status(process_unit_name)))
# Getting process unit default parameters
params = self.__get_process_unit_config__(process_unit_name)
# Checking process unit enable flag
enabled = bool(params.get('enabled', True))
#TODO: it should be add to base of process unit and set_enable command
# should be created.
if enabled == False:
raise BatchProcessManagerException(
'Process unit[%s] is disabled.' % (process_unit_name))
# Updating default parameters with given options
params.update(options)
# Getting process unit instance
process_unit = self.get_process_unit(process_unit_name)
# Configuring the process unit with updated parameters
process_unit.configure(**params)
# Getting joining information
joined_groups = process_unit.get_joined_groups()
joined_units = process_unit.get_joined_units()
# Joining to groups and units
for joined_group in joined_groups:
self.join_to_group(joined_group)
for joined_unit in joined_units:
self.join(joined_unit)
# Getting thread count
thread_count = int(params.get('thread_count', 1))
# Creating a new parallel executor instance
parallel_executor = ParallelExecutor(process_unit.get_name(),
process_unit.process,
thread_count)
# Creating event handler functions
before_start = lambda: self.__before_start_process_unit__(
process_unit, parallel_executor, thread_count, params)
after_finished = lambda: self.__after_finished_process_unit__(
process_unit)
on_failed = lambda error, error_trace: self.__on_proccess_unit_failed__(
process_unit, error, error_trace)
before_start_task = lambda chunk: self.__before_start_process_unit_chunk__(
process_unit, chunk)
after_complete_task = lambda chunk, result: self.__after_complete_process_unit_chunk__(
process_unit, chunk, result)
on_task_failed = lambda chunk, error: self.__on_process_unit_chunk_failed__(
process_unit, chunk, error)
# Setting parallel executor event handler functions
parallel_executor.set_before_start(before_start)
parallel_executor.set_after_finished(after_finished)
parallel_executor.set_on_failed(on_failed)
parallel_executor.set_before_start_task(before_start_task)
parallel_executor.set_after_complete_task(after_complete_task)
parallel_executor.set_on_task_failed(on_task_failed)
# Updating process unit data with created new parallel executor
self.__process_units[process_unit_name] = (process_unit,
parallel_executor)
BatchProcessManager.logger.info(
'Process unit[%s] is creating chunks, its may takes a several minutes...'
% process_unit_name)
# Truncating process unit message stream
process_unit.get_message_stream().truncate()
process_unit.get_message_stream().info(
'Started with parameters[%s]' % params)
# Starting parallel executor to running batch operation
parallel_executor.start()
BatchProcessManager.logger.info(
'Process unit[%s] started with parameters[%s].' %
(process_unit_name, params))
except Exception, error:
message = 'Starting process unit[{0}] failed:{1}'
BatchProcessManager.logger.error(
message.format(process_unit_name, traceback.format_exc()))
if process_unit is not None:
message = 'Starting failed:{0}'
process_unit.get_message_stream().error(
message.format(str(error)))
process_unit.failed(error)
raise
class Communicator(DeltaObject):
'''
The communicator class which is for maintaining listeners.
'''
logger = logging.get_logger(name='communicator')
def __init__(self):
'''
The constructor.
'''
DeltaObject.__init__(self)
self.__factories = {}
self.__listeners = {}
self.__default_listener_name = None
self.__hooks = []
def register_factory(self, type_name, factory):
'''
Registers a listener.
@param listener: listener object which inherited from the Listener class
'''
self.__factories[type_name] = factory
def get_factory(self, type_name):
'''
Return broker factory by broker type name.
@param type_name: type name
@return: CommunicationFactory
'''
if type_name not in self.__factories:
raise CommunicatorException('Factory[%s] not found.' % type_name)
return self.__factories[type_name]
def get_listener(self, name):
'''
Returns the listener by name.
@param name: name of the listener in str
@return: listener[Listener]
'''
# Looking for listener...
if name not in self.__listeners:
raise CommunicatorException('Listener[%s] is not registered.' %
name)
# Getting listener...
listener, thread = self.__listeners[name]
return listener
def get_listeners(self):
"""
Returns all listeners
@return: list<Listener>
"""
return [listener for listener, thread in self.__listeners.values()]
def _get_listener_settings_(self, config_store=None):
'''
Returns listener settings.
@return: dict
'''
listener_settings = {}
if config_store is None:
config_name = "%s.communication" % get_app().get_name()
config_store = config.get_config_store(config_name)
sections = config_store.get_sections()
self.__default_listener_name = config_store.get(
'global', 'default', None)
for name in sections:
if name != 'global':
data = config_store.get_section_data(name)
if not data.has_key('type'):
raise CommunicatorException('Listener[%s] has not type.' %
name)
listener_settings[name] = data
if self.__default_listener_name is None:
self.__default_listener_name = name
return listener_settings
def start(self, config_store=None):
'''
Starts all listeners which is defined in communication configuration.
'''
Communicator.logger.info('Starting communicator.')
try:
listener_settings = self._get_listener_settings_(config_store)
for listener_name in listener_settings:
kwargs = listener_settings[listener_name]
type_name = kwargs['type']
factory = self.get_factory(type_name)
listener = factory.create_listener(self, listener_name, kwargs)
self.__listeners[listener_name] = listener, None
Communicator.logger.info(
'Listener[{lname}] created with params[{lparam}].'.format(
lname=listener_name, lparam=kwargs))
for listener_name in self.__listeners:
listener, thread = self.__listeners[listener_name]
Communicator.logger.info(
'Starting listener[{lname}]...'.format(
lname=listener_name))
thread = run_in_thread(listener.start)
self.__listeners[listener_name] = listener, thread
Communicator.logger.info(
'Listener[{lname}] started.'.format(lname=listener_name))
Communicator.logger.info('Communicator started successfully.')
except Exception:
Communicator.logger.error('Starting communicator failed:%s' %
(traceback.format_exc()))
raise
def stop(self, force=False):
'''
Stops all listeners.
@param force:
'''
try:
Communicator.logger.info('Stopping communicator started.')
threads = []
for listener_name in self.__listeners:
listener, thread = self.__listeners[listener_name]
Communicator.logger.info(
'Stopping listener[{lname}]...'.format(
lname=listener_name))
listener.stop(force)
Communicator.logger.info(
'Listener[{lname}] stopped.'.format(lname=listener_name))
threads.append(thread)
if not force:
for t in threads:
t.join()
Communicator.logger.info('Stopping request processor...')
except Exception:
Communicator.logger.error('Stopping communicator failed:%s' %
(traceback.format_exc()))
raise
def create_proxy_by_ticket(self, ticket, user_name, **kwargs):
'''
Creates a proxy on the given broker type and returns it.
@param ticket: security ticket
@param user_name: user name
@return: Proxy
'''
type_name = kwargs.get('type')
factory = self.get_factory(type_name)
return factory.create_proxy_by_ticket(ticket, user_name, **kwargs)
def create_proxy(self, user_name, password, **kwargs):
'''
Creates a proxy on the given broker type and returns it.
@param user_name: user name
@param password: password
@return: Proxy
'''
type_name = kwargs.get('type')
factory = self.get_factory(type_name)
return factory.create_proxy(user_name, password, **kwargs)
def get_listener_params(self, name):
'''
Returns the listener parameters.
@param name: listener name
@return: dict
'''
return self.get_listener(name).get_params()
def get_default_listener(self):
"""
Returns the default listener.
@return: Listener
"""
return self.get_listener(self.__default_listener_name)
def login(self, listener, login_request):
'''
Logins in application.
@param instance login_request: Client's login request.
@type login_request: Instance of RawRequest.
@return: dict<ticket,
login_date,
data>
'''
if not listener.is_enable():
Communicator.logger.error('Listener[%s] is disabled.' %
listener.get_name())
raise CommunicatorException('Listener[%s] is disabled.' %
listener.get_name())
try:
return request_processor.login(login_request)
except Exception:
Communicator.logger.error(traceback.format_exc())
raise
def logout(self, listener, logout_request):
'''
Log outs the user.
@param instance logout_request: Client's logout request.
@type logout_request: Instance of RawRequest.
'''
if not listener.is_enable():
Communicator.logger.error('Listener[%s] is disabled.' %
listener.get_name())
raise CommunicatorException('Listener[%s] is disabled.' %
listener.get_name())
return request_processor.logout(logout_request)
def execute(self, listener, request, **options):
'''
Executes the given request.
@param listener: listener instance
@param request: client request
@return: object
'''
try:
if not listener.is_enable():
raise CommunicatorException('Listener[%s] is disabled.' %
listener.get_name())
# Processing the request.
return request_processor.process(request, **options)
except Exception:
Communicator.logger.error(traceback.format_exc())
raise
def add_hook(self, hook):
'''
Sets communicator hook.
@param hook: hook
'''
self.__hooks.append(hook)
def get_hooks(self):
'''
Returns communicator hook.
@return: CommunicatorHook
'''
return self.__hooks
import os
import sys
import time
import errno
import threading
import traceback
import multiprocessing
from datetime import datetime
from random import randint
from deltapy.core import DeltaException
from deltapy.multiprocessing.base import BasePool
from deltapy.logging.services import get_logger
LOGGER = get_logger(name='requestprocessor')
class DeltaError(DeltaException):
def __init__(self, message, code=None, data={}, traceback=None):
DeltaException.__init__(self, message)
self._code = self.__class__.__name__
if code is not None:
self._code = code
self._data = data
self._traceback = traceback
def get_code(self):
return self._code
class TransactionCoordinatorCommandExecutionHook(CommandExecutionHook):
'''
Transaction Coordinator Command Execution Hook
'''
LOGGER = logging_services.get_logger(name='request.coordinator')
def before_execute(self, commander, command, *args, **kargs):
'''
This method will be called before command execution.
@param commander: commander
@param command: command
'''
if not self._is_required_to_record_request(command):
return
request = session_services.get_current_session().get_client_request()
coordinator_services.record_request(
command.get_option('recorder_type'), request)
logging_services.debug(
'Request [{0}] for service [{1}] recorded.'.format(
request.id, command.get_key()))
def after_execute(self, commander, command, result, *args, **kargs):
'''
This method will be called after command execution.
@param commander: commander
@param command: command
@param result: command execution result
'''
if not self._is_required_to_record_request(command):
return
request = session_services.get_current_session().get_client_request()
coordinator_services.set_completed(command.get_option('recorder_type'),
request, result)
logging_services.debug(
'Request [{0}] for service [{1}] completed.'.format(
request.id, command.get_key()))
def exception(self, commander, command, error, *args, **kargs):
'''
This method will be called whenever an exception occurred during executing a command.
@param commander: commander
@param command: command
@param error: exception instance
'''
if not self._is_required_to_record_request(command):
return
request = session_services.get_current_session().get_client_request()
coordinator_services.set_failed(command.get_option('recorder_type'),
request, error)
logging_services.debug(
'Request [{0}] for service [{1}] failed.'.format(
request.id, command.get_key()))
def _is_required_to_record_request(self, command):
'''
Returns True if the request must be recorded.
@param Command command: command instance
@rtype: bool
@return: True or False
'''
if not self.is_enable():
return False
recorder_type = command.get_option('recorder_type')
return recorder_type not in (None, '')
class RequestProcessorManager(DeltaObject):
'''
Request processor manager.
'''
logger = logging.get_logger(name='requestprocessor')
def __init__(self):
DeltaObject.__init__(self)
self._processors = {}
self._hooks = []
self._default_processor_name = None
self._event = Event()
self._event.clear()
def register_request_processor_hook(self, hook):
'''
Registers the given hook for request processor manager.
@param RequestProcessorManagerHook hook: hook instance
'''
index = 0
for older_hook in self._hooks:
if older_hook.get_name() == hook.get_name():
self._hooks[index] = hook
return
index += 1
self._hooks.append(hook)
def get_request_processor_hooks(self):
'''
Returns all of the registered hooks.
@rtype: list(RequestProcessorManagerHook)
@return: request processor list
'''
return self._hooks
def register_processor(self, processor):
'''
Registers given processor.
@param processor:
'''
if self._default_processor_name is None:
self.set_default_processor(processor.get_name())
self._processors[processor.get_name()] = processor
def set_default_processor(self, processor_name):
'''
Sets given processor as default processor.
@param processor_name:
'''
self._default_processor_name = processor_name
def get_processors(self):
'''
Returns all registered processors.
'''
return self._processors
def get_processor(self, name=None):
'''
Returns registered processor using given name.
'''
key = self._default_processor_name
if name is not None:
key = name
if key in self._processors:
return self._processors[key]
raise RequestProcessorManagerException(
"Could'nt find processor[{name}] in registered processors.".format(
name=key))
def wait_for_ready(self):
'''
Waits for request processor until it's status be ready.
'''
self._event.wait()
def start(self, **options):
'''
Starts request processor.
@keyword processor_name: Processor to start. If it not provided,
default processor in configs will be used.
@note: Other options will directly pass to the processor.
'''
# Getting configuration store
config_store = config.get_app_config_store('request_processor')
default_processor_name = config_store.get('global', 'default', None)
processor_name = options.get('processor_name')
if processor_name is not None:
self.set_default_processor(processor_name)
options.pop('processor_name')
elif default_processor_name is not None:
self.set_default_processor(default_processor_name)
params = config_store.get_section_data(self._default_processor_name)
params.update(**options)
processor = self.get_processor()
processor.configure(params)
self._event.set()
RequestProcessorManager.logger.info('Request processor started.')
def terminate(self, name=None):
'''
Terminates given processor.
@param name:
'''
RequestProcessorManager.logger.info(
'Terminating request processor [{name}]...'.format(name=name))
processor = self.get_processor(name)
processor.terminate()
RequestProcessorManager.logger.info(
'Request processor [{name}] terminated.'.format(name=name))
def process(self, request, **options):
'''
Processes the request.
@param request: client request
@return: Response
'''
return self.get_processor().process(request, **options)
def login(self, login_request):
"""
Authenticates the given credentials and returns login information.
@param instance login_request: Raw request recevied from client.
@return: login data
@rtype: instance
"""
return self.get_processor().login(login_request)
def logout(self, logout_request):
'''
Logout the user from application.
@param instance logout_request: Raw request received from client.
'''
return self.get_processor().logout(logout_request)
def get_info(self):
'''
Returns information about active request processor.
@return: DynamicObject
'''
processor = self.get_processor()
info = DynamicObject(mode=processor.get_name())
info.update(processor.get_params())
return info
def set_timeout(self, timeout):
'''
Sets global timeout of request processor.
@param timeout: timeout
'''
session = get_current_session()
context = session.get_context()
context['timeout'] = timeout
session.update()
def get_timeout(self):
'''
Returns get global timeout value.
@rtype: int
@return: timeout value
'''
session = get_current_session()
timeout = session.get_context().get('timeout')
if timeout is None:
config_store = \
config_services.get_app_config_store('request_processor')
timeout = config_store.get('global', 'timeout')
if timeout is not None:
timeout = int(timeout)
return timeout
def reload(self):
'''
Re-reads configs from the config file and applying them.
'''
config_store = \
config_services.get_app_config_store('request_processor')
params = config_store.get_section_data(self._default_processor_name)
processor = self.get_processor()
processor.resize(int(params.get("max_processes")))
class ZMQRequestProcessor(RequestProcessorBase):
'''
This processor will process every request in current process.
'''
LOGGER = logging_services.get_logger(name='requestprocessor')
def __init__(self):
RequestProcessorBase.__init__(self, 'zmq')
# Process pool instance
self._pool = None
# ZMQ context
self._context = None
# Facing who sends a request
self._frontend = None
# Facing who processes a request
self._backend = None
# The distributer client
self._dispatcher_pool = None
def configure(self, params):
'''
Configures request processor.
@param params:
'''
# Getting ZMQ parameters
self._params = params
# Getting max processes
max_processes = int(params.get('max_processes', 4))
max_threads = int(params.get('max_threads', 4))
frontend_port = int(params.get('frontend_port'))
backend_port = int(params.get('backend_port'))
# Starting ZMQ Queue
self._context = zmq.Context(1)
self._frontend = self._context.socket(zmq.XREP)
self._frontend.setsockopt(zmq.LINGER, 0)
self._frontend.bind("tcp://*:{0}".format(frontend_port))
self._backend = self._context.socket(zmq.XREQ)
self._backend.setsockopt(zmq.LINGER, 0)
self._backend.bind("tcp://*:{0}".format(backend_port))
self._dispatcher_pool = Queue(max_threads * max_processes)
for i in xrange(max_threads * max_processes):
client = self._context.socket(zmq.REQ)
client.setsockopt(zmq.LINGER, 0)
client.connect("tcp://127.0.0.1:{0}".format(frontend_port))
self._dispatcher_pool.put(client)
def start_queue():
zmq.device(zmq.QUEUE, self._frontend, self._backend)
t = Greenlet(target=start_queue)
t.start()
#gevent.spawn(zmq.device, zmq.QUEUE, self._frontend, self._backend)
# Initializing process pool
self._pool =\
ProcessPool(max_processes,
init_func = ZMQRequestProcessor.init_func)
def process(self, request, **options):
'''
Processes the request.
@param request: client request
@return: Response
'''
client = self._dispatcher_pool.get()
try:
client.send(cPickle.dumps([request, options]))
message = client.recv()
obj = cPickle.loads(message)
if isinstance(obj, Exception):
raise obj
return obj
finally:
self._dispatcher_pool.put(client)
def resize(self, size):
'''
Resizes current process pool.
@param int size: pool size
'''
if self._pool is not None:
return self._pool.set_size(size)
def terminate(self):
'''
Terminates current request processor.
'''
self._pool.terminate()
self._frontend.close()
self._backend.close()
self._context.term()
@staticmethod
def init_func():
settings = config_services.get_app_config_store('request_processor')
backend_port = int(settings.get('zmq', 'backend_port'))
max_threads = int(settings.get('zmq', 'max_threads'))
if max_threads == 0:
max_threads = 4
io_threads = multiprocessing.cpu_count() - 1
if io_threads < 2:
io_threads = 2
ZMQRequestProcessor.LOGGER.info(
"Process [{0}] is started as a subscriber by [{1}] greenlets.".
format(os.getpid(), max_threads))
database_services.reset_pools()
context = zmq.Context(io_threads)
workers = [Worker(context, backend_port) for i in xrange(max_threads)]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
'''
Created on Oct 12, 2009
@author: Abi.Mohammadi & Majid.Vesal
'''
import deltapy.caching.services as caching
from deltapy.utils.decorator import make_decorator
import deltapy.logging.services as logging
import time
logger = logging.get_logger()
@make_decorator
class trace:
'''
Provides caching method result by method input parameters.
'''
logger = logging.get_logger()
def __init__(self, *args, **options):
pass
def wrapper(self, func, *args, **kwargs):
'''
Wraps the function.
@param func: function
@return: depends on function
class ZmqListener(Listener):
'''
ZMQ listener.
'''
logger = logging.get_logger(name = 'ZMQ')
def __init__(self, communicator, name, params, client_request_class=None):
'''
'''
Listener.__init__(self, communicator, name, params, client_request_class=client_request_class)
# Configuring Zmq
workers_count = params.get('workers')
if workers_count is not None:
workers_count = int(workers_count)
else:
workers_count = 16
if not params.has_key('service_name'):
params['service_name'] = self.get_name()
router_url = 'tcp://{0}:{1}'.format(params.get('host'), params.get('port'))
dealer_url = 'inproc://workers'
# Creating zmq context
io_threads = multiprocessing.cpu_count() - 1
if io_threads < 2:
io_threads = 2;
self._context = zmq.Context(io_threads)
# Creating router socket
self._router = self._context.socket(zmq.ROUTER)
self._router.setsockopt(zmq.LINGER, 0)
self._router.bind(router_url)
# Creating dealer socket
self._dealer = self._context.socket(zmq.DEALER)
self._dealer.setsockopt(zmq.LINGER, 0)
self._dealer.bind(dealer_url)
self._dispatchers = []
for index in xrange(workers_count):
dispatcher = ZmqDispatcher(self, self._context, dealer_url)
self._dispatchers.append(dispatcher)
def login(self, ip, user_name, password, **options):
'''
Logins in application.
@param ip: client IP address
@param user_name: user name
@param password: user password
'''
return Listener.login(self, ip, user_name, password, **options)
def login_ex(self, ip, user_name, password, **options):
'''
Logins in application.
@param ip: client IP address
@param user_name: user name
@param password: user password
'''
try:
return Listener.login_ex(self,ip, user_name, password, **options)
except Exception, error:
return make_error_result(self.get_type_convertor(), error)
class Scheduler(Thread):
"""
Provides some functions for scheduling tasks.
"""
logger = logging.get_logger(name = 'scheduler')
def __init__(self):
Thread.__init__(self)
self.setDaemon(True)
self.tasks = {}
self.tasks_lock = Lock()
self.halt_flag = Event()
self.nonempty = Event()
self.on_running_task = lambda t:None
def create_task(self,
name,
start_time,
calc_next_time_func,
func,
**func_kargs):
task = Task(name,
start_time,
calc_next_time_func,
func,
**func_kargs)
return task
def schedule(self,
name,
start_time,
calc_next_time_func,
func,
expire_time = None,
before_run = None,
after_run = None,
retry_count = 0,
**func_args):
task = Task(name,
start_time,
calc_next_time_func,
func,
**func_args)
#task.priority = priority
task.expire_time = expire_time
task.retry_count = retry_count
task.before_run = before_run
task.after_run = after_run
return self.schedule_task(task)
def schedule_task(self, task):
receipt = task.get_id()
try:
self.tasks_lock.acquire()
self.tasks[receipt] = task
self.nonempty.set()
finally:
self.tasks_lock.release()
return receipt
def drop(self, task_receipt):
try:
self.tasks_lock.acquire()
self.tasks[task_receipt].halt()
del self.tasks[task_receipt]
if len(self.tasks)==0:
self.nonempty.clear()
except KeyError:
self.logger.error('Invalid task receipt: %s' % (task_receipt,))
finally:
self.tasks_lock.release()
def halt(self):
self.halt_flag.set()
# Drop all active tasks
map(self.drop, self.tasks.keys())
# Exit the thread to kill the scheduler
def __find_next_tasks__(self):
try:
self.tasks_lock.acquire()
receipts = []
items = self.tasks.items()
by_time = lambda x: operator.getitem(x, 1).scheduled_time
items.sort(key = by_time)
for index in xrange(len(self.tasks)):
receipt = items[index][0]
task = self.tasks[receipt]
task_time = task.scheduled_time
now = datetime.datetime.now()
time_to_wait = task_time - now
secs_to_wait = 0.
secs_to_wait = time_to_wait.seconds
if secs_to_wait <= 0 and secs_to_wait >= -5:
if not task.status in (Task.RUNNING, Task.HALTED):
if task.scheduled_time >= datetime.datetime.now():
receipts.append(receipt)
return receipts
finally:
self.tasks_lock.release()
def run(self):
task_executor = TaskExecutor(self.on_running_task)
while True:
# Waiting for a second
time.sleep(1)
receipts = self.__find_next_tasks__()
ignore_task = False
for receipt in receipts:
if receipt != None:
#self.halt_flag.wait(secs_to_wait)
ignore_task = False
try:
try:
self.tasks_lock.acquire()
task = self.tasks[receipt]
if task.exec_count_limit > 0 and task.exec_count > task.exec_count_limit:
self.logger.debug("Task %s execution count exceeded." % (task.name,))
self.tasks.pop(receipt)
ignore_task = True
#checking task expiration time
if task.expire_time and task.expire_time < datetime.datetime.now():
self.logger.debug("Task %s expired" % (task.name,))
self.tasks.pop(receipt)
ignore_task = True
if not ignore_task:
task_executor.execute(task)
#task.run()
finally:
self.tasks_lock.release()
except Exception, e:
self.logger.exception(e)
self.logger.debug( self.tasks )
else:
self.nonempty.wait()
class TransactionCoordinator(DeltaObject):
LOGGER = logging_services.get_logger(name='request.coordinator')
class StateEnum(DeltaEnum):
RECEIVED = DeltaEnumValue(0, 'Received')
COMPLETED = DeltaEnumValue(1, 'Completed')
FAILED = DeltaEnumValue(2, 'Failed')
REVERSED = DeltaEnumValue(3, 'Reversed')
REVERSE_FAILED = DeltaEnumValue(4, 'Reversed Failed')
def __init__(self):
'''
Initializes transaction coordinator.
'''
DeltaObject.__init__(self)
add_hook(TransactionCoordinatorCommandExecutionHook())
def get_request(self, request_id):
'''
Returns the specified request.
@param str request_id: request ID
@rtype: dict(str request_id: client request ID,
str transaction_id: client request transaction ID,
str user_name: client request user name,
str client_ip: client request IP,
str service_id: client request service ID,
datetime receieve_date: client request recieve date,
datetime request_date: client request request date,
str trace_id: client request trace ID,
int state: request state,
dict data: data)
@type data: dict(dict request_header: request header,
dict command_args: command args,
dict command_kwargs: command kwargs,
dict call_context: call context,
dict response_data: response data
str error: error)
@type request_header: dict(str recorder_type: recorder type,
int version: version)
@type response_data: dict(datetime send_date: client response send date,
dict command_result: client response command result)
@return: request info
'''
return request_recorder_services.get_by_request_id(request_id)
def try_get_request(self, request_id, **options):
'''
Returns information of a particular request.
@param str request_id: request ID
@rtype: dict(str request_id: client request ID,
str transaction_id: client request transaction ID,
str user_name: client request user name,
str client_ip: client request IP,
str service_id: client request service ID,
datetime receieve_date: client request recieve date,
datetime request_date: client request request date,
str trace_id: client request trace ID,
int state: request state,
dict data: data)
@type data: dict(dict request_header: request header,
dict command_args: command args,
dict command_kwargs: command kwargs,
dict call_context: call context,
dict response_data: response data
str error: error)
@type request_header: dict(str recorder_type: recorder type,
int version: version)
@type response_data: dict(datetime send_date: client response send date,
dict command_result: client response command result)
@return: request info
'''
return request_recorder_services.try_get_by_request_id(request_id, **options)
def get_request_state(self, request_id):
'''
Returns state of specified request.
@param str request_id: request ID
@rtype: int
@note:
0: Received
1: Completed
2: Failed
3: Reversed
@return: request state
'''
return request_recorder_services.get_request_state(request_id)
def get_transaction_detail(self, transaction_id):
'''
Returns detail information of the specified transaction.
@param str transaction_id: transaction ID
@rtype: dict(str transaction_id: transaction ID
datetime start_date: start date of transaction,
str user_id: user ID,
list(dict) requests: requests regarding to the transaction)
@type request: dict(str request_id: client request ID,
str transaction_id: client request transaction ID,
str user_name: client request user name,
str client_ip: client request IP,
str service_id: client request service ID,
datetime receieve_date: client request recieve date,
datetime request_date: client request request date,
str trace_id: client request trace ID,
int state: request state,
dict data: data)
@type data: dict(dict request_header: request header,
dict command_args: command args,
dict command_kwargs: command kwargs,
dict call_context: call context,
dict response_data: response data
str error: error)
@type request_header: dict(str recorder_type: recorder type,
int version: version)
@type response_data: dict(datetime send_date: client response send date,
dict command_result: client response command result)
@return: transaction detail
'''
return request_recorder_services.get_detail_by_transaction_id(transaction_id)
def record_request(self, recorder_type, client_request, **options):
'''
Records a request data using the given information.
@param str recorder_type: recorder type
@type client_request: dict(str id: request ID,
str transaction_id: transaction ID,
str user_name: user name,
str client_ip: client IP,
datetime receive_date: receive date,
datetime request_date: request date from client)
@param dict client_request: request data
'''
request_recorder_services.record(recorder_type,
client_request,
**options)
def set_completed(self, recorder_type, client_request, client_response, **options):
'''
Completes the state of the given request.
@param str recorder_type: recorder type
@param dict client_request: request data
@type client_request: dict(str id: request ID,
str transaction_id: transaction ID,
str user_name: user name,
str client_ip: client IP,
datetime receive_date: receive date,
datetime request_date: request date from client)
@param dict client_response: client response
'''
request_recorder_services.set_completed(recorder_type,
client_request,
client_response,
**options)
def set_failed(self, recorder_type, client_request, error, **options):
'''
Sets the state of the given request to failed.
@param str recorder_type: recorder type
@param dict client_request: request data
@type client_request: dict(str id: request ID,
str transaction_id: transaction ID,
str user_name: user name,
str client_ip: client IP,
datetime receive_date: receive date,
datetime request_date: request date from client)
@param duct error: error
'''
request_recorder_services.set_failed(recorder_type,
client_request,
error,
**options)
class TransactionManager(DeltaObject):
DEFAULT_TRANSACTION = 'default'
LOGGER = get_logger(name='transaction')
def __init__(self, database_manager):
DeltaObject.__init__(self)
self._database_manager = database_manager
self._transactions = {}
def get_database_manager(self):
'''
Returns database manager.
@return: DatabaseManager
'''
return self._database_manager
def __add_root_transaction__(self, pool_name, trx):
pool_stack = self._transactions.get(pool_name, [])
pool_stack.append(weakref.ref(trx))
self._transactions[pool_name] = pool_stack
def __remove_root_transaction__(self, pool_name):
pool_stack = self._transactions.get(pool_name, [])
pool_stack.pop()
def __get_current_root_transaction__(self, pool_name):
pool_stack = self._transactions.get(pool_name, [])
if len(pool_stack) > 0:
return pool_stack[len(pool_stack) - 1]()
return None
def begin(self, **kargs):
'''
Begins a transaction and return it.
@param auto_commit: auto commit flag
@param pool_name: connection pool name
@param is_root: root transaction flag
@return: Transaction
'''
auto_commit = kargs.get('auto_commit', False)
pool_name = kargs.get('pool_name',
TransactionManager.DEFAULT_TRANSACTION)
if not pool_name:
pool_name = TransactionManager.DEFAULT_TRANSACTION
is_root = kargs.get('is_root', False)
parent_transaction = None
connection = None
timeout = None
if not is_root:
parent_transaction = self.__get_current_root_transaction__(
pool_name)
if not issubclass(type(parent_transaction), TwoPhaseCommitTransaction):
if parent_transaction is not None:
connection = parent_transaction.get_connection()
else:
connection = self._database_manager.open(pool_name)
trx = Transaction(self, connection, auto_commit, parent_transaction)
trx._pool_name = pool_name
current_session = get_current_session()
if current_session is not None:
client_request = current_session.get_client_request()
if client_request.timeout is not None and client_request.timeout > 0:
timeout = client_request.timeout
self.set_timeout(trx, timeout)
if not parent_transaction:
connection.transaction = trx
self.__add_root_transaction__(pool_name, trx)
return trx
def _run_before_commit_triggers_(self, tx):
'''
'''
triggers = tx.get_before_commit_triggers()
if len(triggers) > 0:
for trigger, params in triggers:
trigger(*params)
def _run_after_commit_triggers_(self, tx):
'''
'''
triggers = tx.get_after_commit_triggers()
if len(triggers) > 0:
for trigger, params in triggers:
trigger(*params)
def commit(self, tx):
try:
if not tx.get_parent():
tx.get_connection().transaction = None
self._run_before_commit_triggers_(tx)
tx.get_connection().commit()
self._run_after_commit_triggers_(tx)
self._database_manager.close(tx.get_connection())
self.__remove_root_transaction__(tx._pool_name)
return True
except Exception:
self.rollback(tx)
raise
return False
def _run_before_rollback_triggers_(self, tx):
'''
'''
triggers = tx.get_before_rollback_triggers()
if len(triggers) > 0:
for trigger, params in triggers:
trigger(*params)
def _run_after_rollback_triggers_(self, tx):
'''
'''
triggers = tx.get_after_rollback_triggers()
if len(triggers) > 0:
for trigger, params in triggers:
trigger(*params)
def rollback(self, tx):
try:
if not tx.get_parent():
tx.get_connection().transaction = None
# Triggers should not interfere with rollback if they're erroneous.
try:
self._run_before_rollback_triggers_(tx)
except Exception as error:
TransactionManager.LOGGER.error(str(error))
tx.get_connection().rollback()
try:
self._run_after_rollback_triggers_(tx)
except Exception as error:
TransactionManager.LOGGER.error(str(error))
self._database_manager.close(tx.get_connection())
self.__remove_root_transaction__(tx._pool_name)
return True
except Exception as error:
TransactionManager.LOGGER.error(str(error))
print str(error)
return False
def tpc_begin(self):
return TwoPhaseCommitTransaction()
def get_current_transaction(self, pool_name=None):
if pool_name is None:
pool_name = TransactionManager.DEFAULT_TRANSACTION
parent_transaction = self.__get_current_root_transaction__(pool_name)
return parent_transaction
def cleanup(self):
for pool_stack in self._transactions.values():
for trx_proxy in pool_stack:
if trx_proxy:
trx = trx_proxy()
if trx:
trx.finalize()
self._transactions.clear()
def set_timeout(self, tx, timeout):
tx.set_timeout(timeout)
def __str__(self):
return "%s" % self._transactions.values()
class CommandManager(DeltaObject):
'''
Manages commands.
'''
logger = logging.get_logger(name='commander')
def __init__(self):
DeltaObject.__init__(self)
self._commands = {}
self._executor = CommandExecutor()
self._hooks = []
def get_executer(self):
'''
Returns defult executor.
'''
return self._executor
def set_executor(self, executor):
'''
Registers a executor by the given name
@param executor: executor instance
'''
self._executor = executor
def execute(self, key, *args, **kargs):
'''
Executes a command by given key.
@param key: command key or command name
@return: object
'''
if key not in self._commands:
raise CommandManagerException("Command[%s] does not exist." % key)
# Getting command
command = self._commands[key]
try:
self._before_execute_command_(command, *args, **kargs)
# Executing the command
result = self._executor.execute(command, *args, **kargs)
self._after_execute_command_(command, result, *args, **kargs)
# Returning the command result
return result
except DeltaException as error:
self._exception_(command, error, *args, **kargs)
error_stack = traceback.format_exc()
error_message = "[{command_key}] Error:\n{error}".format(
command_key=key, error=error_stack)
CommandManager.logger.error(error_message)
raise
except Exception as error:
self._exception_(command, error, *args, **kargs)
error_stack = traceback.format_exc()
error_message = "[{command_key}] Error:\n{error}".format(
command_key=key, error=error_stack)
CommandManager.logger.error(error_message)
# If it's an standard error, there is no need to wrap it again,
# because it can be pickled with no difficulty.
if isinstance(error, StandardError):
raise
raise Exception("%s:%s" % (str(error), error_stack))
def bulk_execute(self, commands, **options):
'''
Executes a command by given key.
@param commands: command data list as dict<command_key, args, kwargs>:
command_key: command key
args: command arguments
kwargs: command keyword arguments
@param **options:
@return: object
'''
if len(commands) == 0:
raise CommandManagerException('There is no command to execute.')
results = []
for command_data in commands:
command_key = command_data['command_key']
args = command_data.get('args', tuple())
if not isinstance(args, (tuple, list)):
args = (args, )
kwargs = command_data.get('kwargs', dict())
result = self.execute(command_key, *args, **kwargs)
results.append(
DynamicObject(command_key=command_key, command_result=result))
return results
def select(self, fields, command_key, *args, **kwargs):
'''
Executes a command and returns requested fields.
@param fields: list of requested fields
@param command_key: command key
@param *args:
@param **kwargs:
@return: [DynamicObject<fields>]
'''
result = self.execute(command_key, *args, **kwargs)
results = result
if not isinstance(result, list):
results = [result]
if fields is None or len(fields) == 0:
return results
selected_resultes = []
for obj in results:
record = DynamicObject()
for field in fields:
record[field] = obj.get(field)
selected_resultes.append(record)
return selected_resultes
def __execute_async__(self, key, callback, *args, **kargs):
result = self.execute(key, *args, **kargs)
if callback:
callback(key, result, *args, **kargs)
return result
def execute_async(self, key, callback, *args, **kargs):
'''
Executes a command by given key.
@param key: command key or command name
@param callback: callback function
@return: None
'''
return run_in_thread(
self.__execute_async__(key, callback, *args, **kargs))
def _before_execute_command_(self, command, *args, **kargs):
'''
Runs before execution method of all hooks.
@param command: command
'''
# Setting start processing time in command manager
current_session = get_current_session()
internal_context = current_session.get_internal_context()
internal_context['start_process_time'] = time.time()
if len(self._hooks):
for hook in self._hooks:
if hook.is_enable():
hook.before_execute(self, command, *args, **kargs)
def _after_execute_command_(self, command, result, *args, **kargs):
'''
Runs after execution method of all hooks.
@param command: command
@param result: command execution result
'''
if len(self._hooks):
for index in xrange(len(self._hooks) - 1, -1, -1):
hook = self._hooks[index]
if hook.is_enable():
hook.after_execute(self, command, result, *args, **kargs)
def _exception_(self, command, error, *args, **kargs):
'''
Runs exception method of all hooks.
@param command: command
@param error: exception instance
'''
if len(self._hooks):
for index in xrange(len(self._hooks) - 1, -1, -1):
hook = self._hooks[index]
if hook.is_enable():
hook.exception(self, command, error, *args, **kargs)
def add_command(self, command, **options):
'''
Adds a command.
@param command: command instance
'''
replace = options.get('replace', False)
if not replace and command.get_key() in self._commands:
raise DeltaException("Command[%s] already exists." % command)
command.attach(self)
self._commands[command.get_key()] = command
def get_commands(self,
parent=None,
name_filter=None,
description_filter=None,
exact_name=None):
'''
Returns all commands.
If any filter specified, it filter those commands.
@keyword str parent: only return commands that their parrents
matches this string.
@keyword str name_filter: only return commands that their names
contain this string.
@keyword str exact_name: only return command that its name
is this string.
@keyword str description_filter: only return commands that their
description contain this string.
@return: founded commands
@rtype: list(dict(str name,
str description))
'''
commands = self._commands.values()
results = []
if (parent is None and name_filter is None
and description_filter is None and exact_name is None):
# If no filter provided.
return commands
for cmd in commands:
filtered = False
if parent is not None:
if not cmd.get_location().startswith(parent):
filtered = True
if name_filter is not None:
if cmd.get_name().find(name_filter) < 0:
filtered = True
if exact_name is not None:
if cmd.get_name() != exact_name:
filtered = True
if description_filter is not None:
command_description = cmd.get_description()
if (command_description is None
or command_description.find(description_filter) < 0):
filtered = True
if not filtered:
results.append(cmd)
return results
def remove_commands(self, parent):
'''
Removes all commands in the given parent domain.
@param parent: parent name
'''
for cmd in self._commands.values():
if cmd.location.find(parent) == 0:
self.remove_command(cmd)
def get_command(self, key):
'''
Returns the command by the given key.
@param key: command key
@return: command
'''
if key in self._commands:
return self._commands[key]
return None
def remove_command(self, command):
'''
Removes the given command.
@param command:
'''
if command.get_key() in self._commands:
command = self._commands.pop(command.get_key())
command.dettach(self)
del command
def add_hook(self, hook):
'''
Adds an execution hook to the commander.
@param hook:
'''
self._hooks.append(hook)
def get_hooks(self):
'''
Returns all execution hooks.
@return: list<CommandExecutionHook>
'''
return self._hooks
class Task:
'''
Task class.
'''
SUCCEEDED = 'succeeded'
FAILED = 'failed'
WAIT = 'wait'
RUNNING = 'running'
HALTED = 'halted'
PRIORITY_LOW = -1
PRIORITY_NORMAL = 0
PRIORITY_HIGH = 1
PRIORITY_VERY_HIGH = 2
logger = logging.get_logger(name = 'scheduler')
def __init__(self,
name,
start_time,
calc_next_time_func,
action,
**kargs):
"""
Initialize a Task.
@param name: Name of task.
@param start_time: First time for task to run
@param calc_next_time_func: Function to calculate the time of next run,
gets one argument, the last run time as a datetime.
Returns None when task should no longer be run
@param action: A function to run
"""
self.name = name
self.start_time = start_time
self.scheduled_time = self.start_time
self.calc_next_time_func = calc_next_time_func
self.action = action
self.action_kargs = kargs
self.retry_count = 0
self.expire_time = None
self.exec_count = 0
self.exec_count_limit = 0
self.priority = Task.PRIORITY_NORMAL
self.halt_flag = Event()
self.task_id = get_uuid()
self.status = Task.WAIT
self.last_exec_time = None
self.last_error = None
self.before_run = lambda task : None
self.after_run = lambda task : None
while self.scheduled_time < datetime.datetime.now():
self.scheduled_time = self.calc_next_time_func(self.scheduled_time)
def run(self):
self.logger.debug("Running Task [%s] , at: %s" % (self.name,
datetime.datetime.now()))
if self.before_run:
self.before_run(self)
retry_count = self.retry_count
if not self.halt_flag.isSet():
while retry_count + 1 > 0:
try:
self.last_exec_time = time.time()
self.action(**self.action_kargs)
self.status = Task.SUCCEEDED
break
except Exception, error:
self.last_error = error
self.status = Task.FAILED
retry_count -= 1
if self.after_run:
self.after_run(self)
self.exec_count += 1
if self.calc_next_time_func:
self.scheduled_time = self.calc_next_time_func(self.scheduled_time)
self.logger.debug("Scheduled next run of %s for: %s" % (self.name,
self.scheduled_time,))
class ChannelAuthenicator(DeltaObject):
'''
Handles channel authentication.
'''
LOGGER = logging_services.get_logger(name='channel')
def _verify_channel(self, channel, **options):
'''
Verifies general conditions on the given channel
@param channel: channel instance
'''
# Verifying channel
if channel is None:
raise ChannelAuthenicationException(_('Channel authentication failed. Please revise your certificate.'))
if not channel.enabled:
raise ChannelAuthenicationException(_('Channel authentication failed. Channel is disable.'))
def authenticate(self, ticket, certificate, **options):
'''
Authenticates a certificate considering the given ticket.
@param ticket: ticket
@param certificate: certificate
'''
# Getting session
session = session_services.get_session(ticket, False)
# Getting current user
current_user = session.get_user()
if channel_services.is_enable():
# Getting login parameters
given_channel_id = options.get('channel')
given_channel = None
if given_channel_id is not None:
given_channel = channel_services.get(given_channel_id)
if not given_channel.certificate_required:
# Verifying channel
self._verify_channel(given_channel)
# Setting channel ID
self._set_channel_to_session(ticket, given_channel_id)
message = 'User [{0}] authenticated through risky channel [{1}]'
ChannelAuthenicator.LOGGER.warning(message.format(current_user.id, given_channel_id))
return
# Getting channel info using certificate
channel = channel_services.try_get_by_certificate(certificate)
# Verifying channel
self._verify_channel(channel)
# Logging
message = 'Authenticating User [{0}] using certification [{1}]'
ChannelAuthenicator.LOGGER.info(message.format(current_user.id, certificate))
if given_channel is not None and given_channel.id != channel.id:
raise ChannelAuthenicationException(_('Channel authentication failed. Channel mismatched.'))
# Setting channel ID
self._set_channel_to_session(ticket, channel.id)
message = 'User [{0}] authenticated with certified channel [{1}]'
ChannelAuthenicator.LOGGER.warning(message.format(current_user.id, channel.id))
def _set_channel_to_session(self, ticket, channel_id, **options):
'''
Sets channel to session using the given ticket.
@param ticket: ticket
@param channel_id: channel ID
'''
# Getting session
session = session_services.get_session(ticket, False)
# Getting session context to set user channel
context = session.get_context()
# Setting channel ID
context['channel'] = channel_id
# Updating session to apply changes
session.update()
from deltapy.core import DeltaObject, DynamicObject
from deltapy.security.authentication.services import authenticate
from deltapy.security.authentication.authenticator import IP_ADDRESS_OPTION_KEY
from deltapy.event_system.decorators import delta_event
from deltapy.request_processor.services import get_timeout
from deltapy.request_processor.response import Response
from deltapy.request_processor.services import get_request_processor_hooks
from deltapy.request_processor.request import ClientRequest
import deltapy.security.authentication.services as authentication_services
import deltapy.commander.services as commander_services
import deltapy.logging.services as logging_services
import deltapy.unique_id.services as unique_id_services
import deltapy.request_processor.helper.services as request_processor_helper_services
LOGGER = logging_services.get_logger(name='requestprocessor')
def handle_request(raw_request, **options):
'''
Processes the request.
@param request: client request
@return: Response
'''
request_dict = raw_request.get_request_dict()
# Converting client request to the type we expected.
type_converter = raw_request.get_converter()
if type_converter is not None:
request_dict = type_converter.to_internal(request_dict)
class BaseAuthenticator(DeltaObject):
'''
'''
INCORRECT_PASSWORD_DELAY = 2
logger = logging.get_logger(name = "authentication")
def _check_password_(self, password, user, **options):
'''
Checks user password and return True if password is correct.
@param password: given password
@param user: user information
@param **options:
@return: bool
'''
raise NotImplementedError()
def _validate_user_(self, user_name, user, **options):
'''
Validates the user.
@param user: user
'''
if not security_services.is_active(user.id):
BaseAuthenticator.logger.error('User [{user_name}] is inactive.'.format(user_name = user_name))
raise UserIsInactiveException(_('User [{user_name}] is inactive.').format(user_name = user_name))
if security_services.is_expired(user.id):
BaseAuthenticator.logger.error('User [{user_name}] is expired.'.format(user_name = user_name))
raise UserIsExpiredException(_('User [{user_name}] is expired.').format(user_name = user_name))
def internal_login(self, user_name, **options):
'''
Logins internally and returns security ticket
@param user_name: user name
@param **options:
@return: object
'''
user = None
try:
user = security_services.get_user(user_name)
except:
force = options.get('force', False)
if force:
user = security_services.create_internal_user(user_name)
else:
BaseAuthenticator.logger.error('User[%s] not found.' % user_name)
message = 'User[{user_name}] not found.'
raise AuthenticationException(message.format(user_name = user_name))
session = session_services.create_internal_session(user)
return session.get_ticket()
def login(self, user_name, password, **options):
'''
Logins and returns security ticket
@param user_name: user name
@param password: user password
@param **options:
@return: object
'''
try:
user = security_services.get_user_by_id(user_name)
if user is None:
BaseAuthenticator.logger.error('User[{user_name}] not found.'.format(user_name = user_name))
raise AuthenticationException(_('User or password is invalid.'))
if not self._check_password_(password, user, **options):
message = 'User[{user_name}] entered invalid password[{password}].'
BaseAuthenticator.logger.error(message.format(user_name=user_name,
password=password))
# Delay to prevent brute force.
time.sleep(BaseAuthenticator.INCORRECT_PASSWORD_DELAY)
raise AuthenticationException(_('User or password is invalid.'))
#Checking the ip address received in options.
self._check_recievied_ip_address(user_name, options)
self._validate_user_(user_name, user, **options)
session = \
session_services.create_session(user,
options.get('client_ip'),
lifetime=options.get('lifetime'))
message = '[{user_name}@{client_ip}] logged in.'
BaseAuthenticator.logger.info(message.format(user_name = user_name,
client_ip = options.get('client_ip')))
return session.get_ticket()
except UserNotFoundException:
BaseAuthenticator.logger.error('User[{user_name}] not found.'.format(user_name = user_name))
time.sleep(BaseAuthenticator.INCORRECT_PASSWORD_DELAY)
raise AuthenticationException(_('User or password is invalid.'))
except Exception as error:
BaseAuthenticator.logger.exception('Exception : {error}'.format(error = error))
raise AuthenticationException(str(error))
def authenticate(self, ticket, user_name, **options):
'''
Authenticates user and ticket.
@param ticket: security ticket
@param user_name: user name
@param **options:
@return: Session
'''
try:
session = session_services.get_session(ticket)
# Checking session expiration
session_services.check_expiration(session)
user = session.get_user()
client_ip = options.get('client_ip')
if user.user_id != user_name:
message = '[{user_name}], [{ip}] sent invalid ticket[{ticket}]'
BaseAuthenticator.logger.error(message.format(user_name=user_name,
ip=client_ip,
ticket=ticket))
raise AuthenticationException(_('The user or ticket is invalid'))
# Checking the ip address received in options.
self._check_recievied_ip_address(user_name, options)
# Checking if ip of session is equal to ip of client.
ip_in_session = session.get_client_ip()
if client_ip != ip_in_session:
if client_ip not in self.get_trusted_ips():
message = '[{user_name}], ticket [{ticket}], came from wrong ip. Session ip [{session_ip}], received ip [{received_ip}].'
BaseAuthenticator.logger.error(message.format(user_name=user_name,
ticket=ticket,
session_ip=ip_in_session,
received_ip=client_ip))
raise AuthenticationException(_('The user or ticket is invalid'))
else:
message = ('User [{user}] Ticket [{ticket}]'
' Received IP [{trusted_ip}] Session IP [{session_ip}],'
' trusted.')
BaseAuthenticator.logger.info(message.format(trusted_ip=client_ip,
user=user_name,
ticket=ticket,
session_ip=ip_in_session))
return session
except Exception:
# Delay to prevent brute force.
time.sleep(BaseAuthenticator.INCORRECT_PASSWORD_DELAY)
raise
def logout(self, ticket, user_name, **options):
'''
Logs off given user.
@param ticket: ticket
@param user_name: user name
@param **options:
'''
try:
message = 'User [{user_name}@{client_ip}] logged off.'
BaseAuthenticator.logger.info(message.format(user_name = user_name,
client_ip = options.get('client_ip')))
session = self.authenticate(ticket, user_name, **options)
session.close()
except:
import traceback
BaseAuthenticator.logger.error(traceback.format_exc())
raise
@cache
def get_trusted_ips(self):
'''
Returns a list of IPs that can be trusted.
@return: List of trusted IPs.
@rtype: list(str)
'''
result = set()
configs = config_services.get_app_config_store()
if configs.has_key('security', 'trusted_ips'):
option_str = configs.get('security', 'trusted_ips')
for trusted_ip in option_str.split(','):
result.add(trusted_ip.strip())
return result
def _check_recievied_ip_address(self, user_name, options):
'''
Checks if the ip_address option passed from client is valid.
If so, replaces the original ip with the received one in the options.
'''
client_ip = options.get('client_ip')
if IP_ADDRESS_OPTION_KEY in options:
if client_ip not in self.get_trusted_ips():
message = 'User [{user_name}] with ip [{client_ip}], sends another ip [{ip_address}], but was not trusted.'
BaseAuthenticator.logger.error(message.format(user_name=user_name,
client_ip=client_ip,
ip_address=options.get(IP_ADDRESS_OPTION_KEY)))
raise AuthenticationException(_("User IP address is invalied."))
message = 'User [{user}] came from trusted ip [{trusted_ip}]. True IP: [{true_ip}].'
BaseAuthenticator.logger.info(message.format(user=user_name,
trusted_ip=client_ip,
true_ip=options.get(IP_ADDRESS_OPTION_KEY)))
options.update(client_ip=options.pop(IP_ADDRESS_OPTION_KEY))
class ServerStatusReporter(object):
LOGGER = get_logger(name='status_reporter')
def run(self):
'''
Starts the reporter.
'''
run_in_thread(self._listen)
def _listen(self):
'''
Listens on a socket.
'''
server_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# For re-using a time-wait socket.
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind('/var/run/{0}/{1}.sid'.format(
application_services.get_name(), os.getpid()))
server_socket.listen(1)
while True:
try:
connection, address = server_socket.accept()
status_string = self._get_status()
# Since we don't care how much data has been send before an
# error happends, we use `sendall' instead of `send'.
connection.sendall(status_string)
except Exception as error:
ServerStatusReporter.LOGGER.error(str(error))
ServerStatusReporter.LOGGER.error(traceback.format_exc())
finally:
connection.close()
def _get_status(self):
'''
Gets status of the system.
@return: status of system as a string.
@rtype: str
'''
# Gathering server status.
status_string = ''
status_string += 'Instance Name: {0}\n'.format(
application_services.get_full_name())
# Application name and version
app_introduction = application_services.introduce()
if 'name' in app_introduction:
status_string += 'Name: {0}\n'.format(app_introduction['name'])
if 'version' in app_introduction:
status_string += 'Version: {0}\n'.format(
app_introduction['version'])
# Packages status
loaded_packages_count = len(package_services.get_loaded_packages())
disabled_packages = package_services.get_disabled_packages()
status_string += 'Packages: Loaded={0}, Disabled={1}\n'.format(
loaded_packages_count, len(disabled_packages))
if len(disabled_packages) > 0:
status_string += 'Disabled Packages: [{0}'.format(
disabled_packages[0])
for disabled_package in disabled_packages[1:]:
status_string += ',\n'
status_string += ' ' * 20
status_string += disabled_package
status_string += ']\n'
# Request processor status
info = request_processor_services.get_info()
status_string += 'Request Processor Parameters: {0}\n'.format(
str(info))
# Threads/Processes
current_pid = os.getpid()
current_process_thread_count = None
# Reading current process' thread count.
with open('/proc/{0}/status'.format(current_pid), 'r') as status_file:
while True:
file_line = status_file.readline()
if not file_line:
break
if file_line.startswith('Threads:'):
current_process_thread_count = int(file_line[8:].strip())
break
status_string += 'Master Process Threads: {0}\n'.format(
current_process_thread_count)
# Finding all child processes.
total_threads = 0
total_children = 0
for process_dir in os.listdir('/proc/'):
if process_dir.isdigit() and os.path.isdir(
'/proc/{0}'.format(process_dir)):
with open('/proc/{0}/status'.format(process_dir),
'r') as status_file:
while True:
file_line = status_file.readline()
if not file_line:
break
if file_line.startswith('PPid:'):
if int(file_line[5:].strip()) == current_pid:
# It's our child.
total_children += 1
# Continue to find threads.
while True:
file_line = status_file.readline()
if not file_line:
break
if file_line.startswith('Threads:'):
total_threads += int(
file_line[8:].strip())
break
break
status_string += 'Child Processes: {0}\n'.format(total_children)
status_string += 'Child Threads: {0}\n'.format(total_threads)
# TODO: Add database connections here.
# Sessions
status_string += 'Sessions: {0}'.format(
session_services.get_sessions_count())
return status_string
class ChannelAuthorizer(DeltaObject):
'''
It is responsible to provide functionality to limit services on user channels.
'''
LOGGER = logging_services.get_logger(name='channel')
GLOBAL_ALLOWED = ['app.introduce',
'command.list',
'command.doc',
'security.authenticate',
'request_processor.coordinator.request.state']
def __init__(self):
DeltaObject.__init__(self)
commander_services.add_hook(ChannelAuthorizerCommandHook())
def _get_commands(self, pattern, all_commands):
'''
Returns list of commands according to the given pattern.
@param pattern: pattern
@rtype: [str]
@return: commands
'''
result = []
for command in all_commands:
if fnmatch.fnmatch(command, pattern.strip()):
result.append(command)
return result
def _get_allowed_commands(self, channel_id):
'''
Returns allowed commands on the given channel.
@param channel_id: channel ID
@rtype: [str]
@return: allowed commands
'''
channel = channel_services.get(channel_id)
if channel.allowed is None or len(channel.allowed.strip()) == 0:
channel.allowed = '*'
allowed_commands = channel.allowed.replace(';', ',').split(',')
result = []
for pattern in allowed_commands:
result.extend(self._get_commands(pattern, self._get_all_commands()))
return set(result)
def _get_denied_commands(self, channel_id):
'''
Returns denied commands on the given channel.
@param channel_id: channel ID
@rtype: list<str>
@return: denied commands
'''
channel = channel_services.get(channel_id)
if channel.denied is None or len(channel.denied.strip()) == 0:
return set([])
denied_commands = channel.denied.replace(';', ',').split(',')
result = []
for pattern in denied_commands:
result.extend(self._get_commands(pattern, self._get_all_commands()))
return set(result)
def _get_all_commands(self):
'''
Returns all of application commands
@rtype: [str]
@return: command list
'''
result = []
for command in commander_services.get_commands():
result.append(command.get_key())
return result
@cache
def get_channel_authorized_commands(self, channel_id):
'''
Returns all authorized command on the given channel.
@param channel_id: channel ID
'''
allowed = set(self._get_allowed_commands(channel_id))
#ChannelAuthorizer.LOGGER.debug('allowed:{0}'.format(allowed))
denied = set(self._get_denied_commands(channel_id))
#ChannelAuthorizer.LOGGER.debug('denied:{0}'.format(denied))
return allowed.difference(denied).union(set(ChannelAuthorizer.GLOBAL_ALLOWED))
def authorize(self, channel_id, command_key, **options):
'''
Authorizes the given command on the specified channel.
@param channel_id: channel ID
@param command_key: command key
'''
if channel_services.is_enable():
# Getting command instance
command = commander_services.get_command(command_key)
if command is None:
message = _('Command [{0}] not found.')
raise ChannelAuthorizerException(message.format(command_key))
ChannelAuthorizer.LOGGER.debug('Authorizing command [{0}] on channel [{1}]'.format(command_key, channel_id))
if command.get_key() not in self.get_channel_authorized_commands(channel_id):
message = _('Channel [{0}] has not access on command [{1}].')
raise ChannelAccessDenied(message.format(channel_id, command_key))
ChannelAuthorizer.LOGGER.info('Command [{0}] authorized on channel [{1}]'.format(command_key, channel_id))
def get_command_list(self, parent = None):
'''
Returns available commands list.
@param parent: parent command
@rtype: [str]
@return: command list
'''
result = []
for command in commander_services.get_commands(parent):
result.append(command.get_key())
if not channel_services.is_enable():
return result
allowed = self.get_channel_authorized_commands(get_current_channel_id())
return list(set(allowed).intersection(set(result)))
def get_command_doc(self, key):
'''
Returns defined document for the specified command.
@param key: command key
@rtype: str
@return: command document
'''
# Getting command instance
command = commander_services.get_command(key)
if command is None:
message = _('Command [{0}] not found.')
raise ChannelAuthorizerException(message.format(key))
if channel_services.is_enable():
# Authorizing the command
self.authorize(get_current_channel_id(), key)
return command.get_doc()
class TypeConvertor(NullTypeConvertor):
'''
Provides functionality for converting external types to internal types and vice versa
'''
LOGGER = get_logger(name='root')
def __init__(self):
'''
Initializes type converter.
'''
NullTypeConvertor.__init__(self)
self.__internal_traverser = ObjectTraverser(self.safe_internal_convert)
self.__external_traverser = ObjectTraverser(self.safe_external_convert)
self._invalid_phrases = \
['`', '^', '\\', 'script', 'html', 'xhtml', 'colon', 'base64',
'import', 'exec', 'eval', 'compile', 'getattr', 'setattr',
'__getattr__', '__getattribute__', '__setattr__', '__delattr__',
'__class__', '__module__', 'BANNED_WORDS']
self._invalid_regex = \
[r'[\s\S]*<[\s\S]*>[\s\S]*',
r'[\s\S]*[&#"\'][\s\S]*;[\s\S]*',
r'([\s\"\'`;\/0-9\=]+on\w+\s*=)']
self._combined_invalid_regex = \
re.compile("(" + ")|(".join(self._invalid_regex) + ")")
def _normalize(self, obj):
return obj.lower().strip().replace('\n', '').replace('\r', '')
def _validate(self, obj):
if not isinstance(obj, (str, unicode)):
return
normalized_obj = self._normalize(obj)
for invalid_phrase in self._invalid_phrases:
if invalid_phrase in normalized_obj:
raise TypeConverterException(_('Invalid phrase [{0}] got detected.'.format(invalid_phrase)))
if re.match(self._combined_invalid_regex, normalized_obj):
raise TypeConverterException(_('Invalid phrase [{0}] got detected.'.format(obj)))
def safe_internal_convert(self, obj):
#self._validate(obj)
return self.internal_convert(obj)
def safe_external_convert(self, obj):
return self.external_convert(obj)
def internal_convert(self, obj):
return obj
def external_convert(self, obj):
return obj
def to_internal(self, obj):
'''
Converts an extranal object type to internal type.
@param obj: external object
@return: object
'''
return self.__internal_traverser.update(obj)
def to_external(self, obj):
'''
Converts an internal object type to external object type.
@param obj: internal object
@return: object
'''
return self.__external_traverser.update(obj)
class IceDispatcher(DeltaIce.IIceDispatcher, Dispatcher):
logger = logging.get_logger(name='communicator')
def __init__(self, listener):
Dispatcher.__init__(self, listener)
def _get_client_ip_(self, **options):
current = options.get('current')
if current is not None:
ip_port = current.con.toString().split('remote address = ')[1]
ip_address = ip_port.split(':')[0]
return utils.object_to_dobject(ip_address)
def login(self, userName, password, options, current=None):
try:
start = time.time()
login_result = \
self._listener.login(self._get_client_ip_(current=current),
userName,
password,
**options)
end = time.time()
ticket = login_result.value['ticket'].value
time_span = end - start
IceDispatcher.logger.info("Ice [{0}] login [{1}]".format(
ticket, time_span))
return login_result
except DeltaException as error:
exception = DeltaIce.AuthenticationException()
exception.code = error.get_code()
exception.data = utils.object_to_dobject(error.get_data())
exception.message = utils.str_to_external(str(error))
# Getting the true trace back from the error itself.
if error.get_traceback() is not None:
exception.traceback = utils.str_to_external(
error.get_traceback())
else:
exception.traceback = utils.str_to_external(
traceback.format_exc())
raise exception
except Exception as error:
exception = DeltaIce.GenericException()
exception.message = utils.str_to_external(str(error))
# Getting the true trace back from the error itself.
if hasattr(error, 'traceback'):
exception.traceback = getattr(error, 'traceback')
else:
exception.traceback = traceback.format_exc()
raise exception
def loginEx(self, userName, password, options, current=None):
try:
start = time.time()
login_result = \
self._listener.login_ex(self._get_client_ip_(current=current),
userName,
password,
**options)
end = time.time()
ticket = login_result.value['ticket'].value
time_span = end - start
IceDispatcher.logger.info("Ice [{0}] login [{1}]".format(
ticket, time_span))
return login_result
except DeltaException as error:
exception = DeltaIce.AuthenticationException()
exception.code = error.get_code()
exception.data = utils.object_to_dobject(error.get_data())
exception.message = utils.str_to_external(str(error))
# Getting the true trace back from the error itself.
if error.get_traceback() is not None:
exception.traceback = utils.str_to_external(
error.get_traceback())
else:
exception.traceback = utils.str_to_external(
traceback.format_exc())
raise exception
except Exception as error:
exception = DeltaIce.GenericException()
exception.message = utils.str_to_external(str(error))
# Getting the true trace back from the error itself.
if hasattr(error, 'traceback'):
exception.traceback = getattr(error, 'traceback')
else:
exception.traceback = traceback.format_exc()
raise exception
def logout(self, ticket, userName, current=None):
try:
start = time.time()
logout_result = \
self._listener.logout(self._get_client_ip_(current=current),
ticket,
userName)
end = time.time()
time_span = end - start
IceDispatcher.logger.info("Ice [{0}] logout [{1}]".format(
ticket.value, time_span))
return logout_result
except Exception as error:
exception = DeltaIce.GenericException()
exception.message = str(error)
# Getting the true trace back from the error itself.
if hasattr(error, 'traceback'):
exception.traceback = getattr(error, 'traceback')
else:
exception.traceback = traceback.format_exc()
raise exception
def execute(self,
ticket,
userName,
commandKey,
args,
kwargs,
current=None):
try:
execute_result = \
self._listener.execute(self._get_client_ip_(current=current),
ticket,
userName,
commandKey,
*args,
**kwargs)
return execute_result
except DeltaException as error:
exception = DeltaIce.GenericException()
exception.code = error.get_code()
exception.data = utils.object_to_dobject(error.get_data())
exception.message = utils.str_to_external(str(error))
# Getting the true trace back from the error itself.
if error.get_traceback() is not None:
exception.traceback = utils.str_to_external(
error.get_traceback())
else:
exception.traceback = utils.str_to_external(
traceback.format_exc())
raise exception
except Exception as error:
exception = DeltaIce.GenericException()
exception.message = utils.str_to_external(str(error))
# Getting the true trace back from the error itself.
if hasattr(error, 'traceback'):
exception.traceback = getattr(error, 'traceback')
else:
exception.traceback = traceback.format_exc()
raise exception
def executeEx(self, request, current=None):
try:
start = time.time()
request['ip'] = self._get_client_ip_(current=current)
execute_result = \
self._listener.execute_ex(request)
end = time.time()
request_id = execute_result.value['request_id'].value
time_span = end - start
IceDispatcher.logger.info("Ice [{0}] executed [{1}]".format(
request_id, time_span))
return execute_result
except DeltaException, error:
exception = DeltaIce.GenericException()
exception.code = error.get_code()
exception.data = utils.object_to_dobject(error.get_data())
exception.message = utils.str_to_external(str(error))
# Getting the true trace back from the error itself.
if error.get_traceback() is not None:
exception.traceback = utils.str_to_external(
error.get_traceback())
else:
exception.traceback = utils.str_to_external(
traceback.format_exc())
raise exception
except Exception, error:
exception = DeltaIce.GenericException()
exception.message = utils.str_to_external(str(error))
# Getting the true trace back from the error itself.
if hasattr(error, 'traceback'):
exception.traceback = getattr(error, 'traceback')
else:
exception.traceback = traceback.format_exc()
raise exception
class IceJsonListener(Listener):
#logger = logging.get_logger(name = 'IceDispatcher')
communicator_logger = logging.get_logger(name='communicator')
def __init__(self, communicator, name, params, client_request_class=None):
Listener.__init__(self,
communicator,
name,
params,
client_request_class=client_request_class)
if not params.has_key('service_name'):
params['service_name'] = self.get_name()
host = params.get('host')
port = int(params.get('port'))
protocol = params.get('protocol')
if protocol is None:
protocol = 'tcp'
ice_url = "%s -h %s -p %d" % (protocol, host, port)
properties = Ice.createProperties()
for property_name, property_value in params.iteritems():
#Passing configs to Ice as properties. Ice will pick its
#own configs and ignore others.
properties.setProperty(property_name, property_value)
data = Ice.InitializationData()
data.properties = properties
self._proxy = Ice.initialize(sys.argv, data)
self._adapter = self._proxy.createObjectAdapterWithEndpoints(
get_app().get_name(), ice_url)
self.add_service(IceJsonDispatcher(self), name=params['service_name'])
def add_service(self, service, **kwargs):
'''
Add a service to listener.
@param service: service object
'''
name = kwargs.get('name', None)
self._adapter.add(service, self._proxy.stringToIdentity(name))
self._adapter.activate()
def start(self):
self._proxy.waitForShutdown()
def stop(self, force):
self._proxy.destroy()
def login(self, ip, user_name, password, **options):
'''
Logins in application.
@param ip: client IP address
@param user_name: user name
@param password: user password
@note: This method is existed for backward compability.
`login_ex' should be used instead.
'''
start = time.time()
login_result = \
super(IceJsonListener, self).login(ip, user_name, password, **options)
end = time.time()
time_span = end - start
IceJsonListener.communicator_logger.info(
"Ice-Json [{0}] login [{1}]".format(login_result, time_span))
return login_result
def logout(self, ip, ticket, user_name):
'''
Log outs the user.
@param ip: client ip
@param ticket: user ticket
@param user_name: user name
'''
start = time.time()
logout_result = \
super(IceJsonListener, self).logout(ip, ticket, user_name)
end = time.time()
time_span = end - start
IceJsonListener.communicator_logger.info(
"Ice-Json [{0}] logout [{1}]".format(ticket, time_span))
return logout_result
def execute_ex(self, request, **options):
'''
Executes the given request.
@param request: client request
@return: object
'''
start = time.time()
execute_result = \
super(IceJsonListener, self).execute_ex(request, **options)
end = time.time()
request_id = execute_result["request_id"]
time_span = end - start
IceJsonListener.communicator_logger.info(
"Ice-Json [{0}] executed [{1}]".format(request_id, time_span))
return execute_result
class ChannelManager(DeltaObject):
'''
Provides basic functionality to manipulate channels.
'''
LOGGER = logging_services.get_logger(name='channel')
def __init__(self):
'''
'''
DeltaObject.__init__(self)
self._channels = {}
self._enabled = None
def _get_certificate(self, channel_id, **options):
'''
Returns certificate content.
@param channel_id: cannel id.
@param **options:
certificate_required: determines whether certificate verification is required. (Default is True)
@rtype: string
@return: certificate content.
'''
file_name = '{0}.cert'.format(channel_id).lower()
settings_path = os.path.join(get_application_dir(), 'settings')
certificate_path = os.path.join(settings_path, 'certificates')
certificate_file = os.path.join(certificate_path, file_name)
certificate_required = options.get('certificate_required')
if certificate_required is None:
certificate_required = True
message = 'Loading certificate file [{0}]'
ChannelManager.LOGGER.debug(message.format(certificate_file))
if not os.path.exists(certificate_file):
message = 'Checking certificate required flag is [{0}] for channel [{1}]'
ChannelManager.LOGGER.debug(
message.format(certificate_required, channel_id))
if certificate_required:
message = 'Certificate file [{0}] not found.'
raise CertificateFileNotFoundException(
message.format(certificate_file))
return unique_id_services.get_id('uuid')
certificate = open(certificate_file, 'rb').read()
return certificate
def enable_channel(self, channel_id, flag):
'''
Enables or disables a channel.
@param channel_id: channel ID
@param flag: True or False
'''
message = 'Trying to enable channel [{0}]'
if not flag:
message = 'Trying to disable channel [{0}]'
ChannelManager.LOGGER.debug(message.format(channel_id))
# Getting channel information
channel = self.get(channel_id)
# Setting flag
channel.enabled = flag
message = 'Channel [{0}] enabled successfully'
if not flag:
message = 'Channel [{0}] disabled successfully'
ChannelManager.LOGGER.info(message.format(channel_id))
def is_registered(self, channel_id):
'''
Returns True if the given channel is already registered.
@param channel_id: channel ID
@rtype: bool
@return: True or False
'''
return channel_id in self._channels
def get(self, channel_id):
'''
Returns channel information according to the given channel ID.
@param channel_id: channel ID
@rtype: DynamicObject<id,
description,
certificate,
certificate_required,
enabled>
@return: channel information
'''
channel = self._channels.get(channel_id)
if channel is None:
message = _('Channel [{0}] is not registered.')
raise ChannelIsNotRegisteredException(message.format(channel_id))
return channel
def try_get_by_certificate(self, certificate):
'''
Returns channel information corresponded to the given certificate.
@param certificate: certificate
@rtype: DynamicObject<id,
description,
certificate,
certificate_required,
enabled>
@return: channel information
'''
for channel in self._channels.values():
if certificate is not None and \
channel.certificate.strip() == certificate.strip():
return channel
return None
def get_all(self, **options):
'''
Returns information of all channels.
@rtype: [DynamicObject<id,
description,
certificate,
certificate_required,
enabled>]
@return: all channels
'''
return self._channels.values()
@delta_event('register_channel')
def register(self, channel_id, certificate, **options):
'''
Registers a new channel.
@param channel_id: channel ID
@param certificate: certificate content
@param **options:
description: channel description
enabled: channel enable flag
'''
message = 'Trying to register channel [{0}]'
ChannelManager.LOGGER.debug(message.format(channel_id))
# Checking channel existence
if self.is_registered(channel_id):
message = _('Channel {0} is already registered.')
raise ChannelManagerException(message.format(channel_id))
# Checking the previous channel with the same certificate
registered_channel = \
self.try_get_by_certificate(certificate)
if registered_channel is not None:
message = _('Certificate is in use.')
raise ChannelManagerException(message)
# Deciding on important parameters
options['description'] = options.get('description')
options['enabled'] = options.get('enabled')
if options['enabled'] is None:
options['enabled'] = True
options['certificate_required'] = options.get('certificate_required')
if options['certificate_required'] is None:
options['certificate_required'] = True
# Setting allowed and denied commands
options['allowed'] = options.get('allowed')
options['denied'] = options.get('denied')
# Creating channel information
channel = DynamicObject(id=channel_id, certificate=certificate)
channel.update(options)
# Registering channel
self._channels[channel_id] = channel
message = 'Channel [{0}] successfully registered'
ChannelManager.LOGGER.info(message.format(channel_id))
def unregister(self, channel_id, **options):
'''
Unregisters a channel.
@param channel_id: channel ID
'''
message = 'Trying to unregister channel [{0}]'
ChannelManager.LOGGER.debug(message.format(channel_id))
# Getting channel information
channel = self.get(channel_id)
# Unregistering the channel
self._channels.pop(channel.id)
message = 'Channel [{0}] successfully unregistered'
ChannelManager.LOGGER.info(message.format(channel_id))
def get_current_channel_id(self):
'''
Returns current channel that user logged in with.
@rtype: str
@return: channel ID
'''
session = get_current_session()
if session is not None:
context = session.get_context()
return context.get('channel')
return None
def load(self):
'''
Loads all channels.
'''
try:
message = 'Loading channels'
ChannelManager.LOGGER.debug(message)
config_store = config_services.get_app_config_store('channel')
self._enabled = config_store.get('global', 'enabled')
if self._enabled is not None:
self._enabled = eval(self._enabled)
else:
self._enabled = True
if self._enabled:
for section in config_store.get_sections():
if section != 'global':
section_data = config_store.get_section_data(section)
channel_id = section
for key in section_data:
section_data[key] = eval(str(section_data[key]))
certificate = self._get_certificate(
channel_id, **section_data)
self.register(channel_id, certificate, **section_data)
else:
message = 'Channel manager is disabled'
ChannelManager.LOGGER.info(message)
except ConfigFileNotFoundException:
message = 'Channel settings not found and channel manager is disabled'
ChannelManager.LOGGER.info(message)
self._enabled = False
def is_enable(self):
'''
Returns True if the channel manager is enable.
@rtype: bool
@return: True or False
'''
return self._enabled
