def __init__(self):
self.logger = logging.getLogger('middleware')
self.__jobs = JobsQueue(self)
self.__schemas = {}
self.__services = {}
self.__wsclients = {}
self.__init_services()
self.__init_rollbar()
self.__plugins_load()
def __init__(self):
self.logger_name = logger.Logger('middlewared')
self.logger = self.logger_name.getLogger()
self.rollbar = logger.Rollbar()
self.__jobs = JobsQueue(self)
self.__schemas = {}
self.__services = {}
self.__wsclients = {}
self.__hooks = defaultdict(list)
self.__server_threads = []
self.__init_services()
self.__plugins_load()
def __init__(self):
self.logger = logging.getLogger('middleware')
self.__jobs = JobsQueue()
self.__schemas = {}
self.__services = {}
self.__init_services()
self.__init_rollbar()
self.__plugins_load()
class Middleware(object):
def __init__(self):
self.logger_name = logger.Logger('middlewared')
self.logger = self.logger_name.getLogger()
self.rollbar = logger.Rollbar()
self.__jobs = JobsQueue(self)
self.__schemas = {}
self.__services = {}
self.__wsclients = {}
self.__hooks = defaultdict(list)
self.__server_threads = []
self.__init_services()
self.__plugins_load()
def __init_services(self):
from middlewared.service import CoreService
self.add_service(CoreService(self))
def __plugins_load(self):
from middlewared.service import Service, CRUDService, ConfigService
plugins_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'plugins',
)
self.logger.debug('Loading plugins from {0}'.format(plugins_dir))
if not os.path.exists(plugins_dir):
raise ValueError('plugins dir not found')
for f in os.listdir(plugins_dir):
if not f.endswith('.py'):
continue
f = f[:-3]
fp, pathname, description = imp.find_module(f, [plugins_dir])
try:
mod = imp.load_module(f, fp, pathname, description)
finally:
if fp:
fp.close()
for attr in dir(mod):
attr = getattr(mod, attr)
if not inspect.isclass(attr):
continue
if attr in (Service, CRUDService, ConfigService):
continue
if issubclass(attr, Service):
self.add_service(attr(self))
if hasattr(mod, 'setup'):
mod.setup(self)
# Now that all plugins have been loaded we can resolve all method params
# to make sure every schema is patched and references match
from middlewared.schema import resolver # Lazy import so namespace match
to_resolve = []
for service in self.__services.values():
for attr in dir(service):
to_resolve.append(getattr(service, attr))
resolved = 0
while len(to_resolve) > 0:
for method in list(to_resolve):
try:
resolver(self, method)
except ValueError:
pass
else:
to_resolve.remove(method)
resolved += 1
if resolved == 0:
raise ValueError("Not all could be resolved")
self.logger.debug('All plugins loaded')
def register_wsclient(self, client):
self.__wsclients[client.sessionid] = client
def unregister_wsclient(self, client):
self.__wsclients.pop(client.sessionid)
def register_hook(self, name, method, sync=True):
"""
Register a hook under `name`.
The given `method` will be called whenever using call_hook.
Args:
name(str): name of the hook, e.g. service.hook_name
method(callable): method to be called
sync(bool): whether the method should be called in a sync way
"""
self.__hooks[name].append({
'method': method,
'sync': sync,
})
def call_hook(self, name, *args, **kwargs):
"""
Call all hooks registered under `name` passing *args and **kwargs.
Args:
name(str): name of the hook, e.g. service.hook_name
"""
for hook in self.__hooks[name]:
try:
if hook['sync']:
hook['method'](*args, **kwargs)
else:
gevent.spawn(hook['method'], *args, **kwargs)
except:
self.logger.error('Failed to run hook {}:{}(*{}, **{})'.format(name, hook['method'], args, kwargs), exc_info=True)
def add_service(self, service):
self.__services[service._config.namespace] = service
def get_service(self, name):
return self.__services[name]
def get_services(self):
return self.__services
def add_schema(self, schema):
if schema.name in self.__schemas:
raise ValueError('Schema "{0}" is already registered'.format(
schema.name
))
self.__schemas[schema.name] = schema
def get_schema(self, name):
return self.__schemas.get(name)
def get_jobs(self):
return self.__jobs
def call_method(self, app, message):
"""Call method from websocket"""
params = message.get('params') or []
service, method_name = message['method'].rsplit('.', 1)
methodobj = getattr(self.get_service(service), method_name)
if not app.authenticated and not hasattr(methodobj, '_no_auth_required'):
app.send_error(message, 'Not authenticated')
return
args = []
if hasattr(methodobj, '_pass_app'):
args.append(app)
# If the method is marked as a @job we need to create a new
# entry to keep track of its state.
job_options = getattr(methodobj, '_job', None)
if job_options:
# Create a job instance with required args
job = Job(self, message['method'], methodobj, args, job_options)
# Add the job to the queue.
# At this point an `id` is assinged to the job.
self.__jobs.add(job)
else:
job = None
args.extend(params)
if job:
return job.id
else:
return methodobj(*args)
def call(self, method, *params):
service, method = method.rsplit('.', 1)
return getattr(self.get_service(service), method)(*params)
def send_event(self, name, event_type, **kwargs):
assert event_type in ('ADDED', 'CHANGED', 'REMOVED')
for sessionid, wsclient in self.__wsclients.iteritems():
try:
wsclient.send_event(name, event_type, **kwargs)
except:
self.logger.warn('Failed to send event {} to {}'.format(name, sessionid), exc_info=True)
def pdb(self):
import pdb
pdb.set_trace()
def run(self):
gevent.signal(signal.SIGTERM, self.kill)
gevent.signal(signal.SIGUSR1, self.pdb)
Application.middleware = self
wsserver = WebSocketServer(('127.0.0.1', 6000), Resource(OrderedDict([
('/websocket', Application),
])))
restful_api = RESTfulAPI(self)
apidocs_app.middleware = self
apidocsserver = WSGIServer(('127.0.0.1', 8001), apidocs_app)
restserver = WSGIServer(('127.0.0.1', 8002), restful_api.get_app())
self.__server_threads = [
gevent.spawn(wsserver.serve_forever),
gevent.spawn(apidocsserver.serve_forever),
gevent.spawn(restserver.serve_forever),
gevent.spawn(self.__jobs.run),
]
self.logger.debug('Accepting connections')
gevent.joinall(self.__server_threads)
def kill(self):
self.logger.info('Killall server threads')
gevent.killall(self.__server_threads)
sys.exit(0)
class Middleware(object):
def __init__(self):
self.logger = logging.getLogger('middleware')
self.__jobs = JobsQueue()
self.__schemas = {}
self.__services = {}
self.__init_services()
self.__init_rollbar()
self.__plugins_load()
def __init_services(self):
from middlewared.service import CoreService
self.add_service(CoreService(self))
def __init_rollbar(self):
rollbar.init(
'caf06383cba14d5893c4f4d0a40c33a9',
'production' if 'DEVELOPER_MODE' not in os.environ else 'development'
)
def __plugins_load(self):
from middlewared.service import Service, CRUDService, ConfigService
plugins_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'plugins',
)
self.logger.debug('Loading plugins from {0}'.format(plugins_dir))
if not os.path.exists(plugins_dir):
raise ValueError('plugins dir not found')
for f in os.listdir(plugins_dir):
if not f.endswith('.py'):
continue
f = f[:-3]
fp, pathname, description = imp.find_module(f, [plugins_dir])
try:
mod = imp.load_module(f, fp, pathname, description)
finally:
if fp:
fp.close()
for attr in dir(mod):
attr = getattr(mod, attr)
if not inspect.isclass(attr):
continue
if attr in (Service, CRUDService, ConfigService):
continue
if issubclass(attr, Service):
self.add_service(attr(self))
if hasattr(mod, 'setup'):
mod.setup(self)
# Now that all plugins have been loaded we can resolve all method params
# to make sure every schema is patched and references match
from middlewared.schema import resolver # Lazy import so namespace match
to_resolve = []
for service in self.__services.values():
for attr in dir(service):
to_resolve.append(getattr(service, attr))
resolved = 0
while len(to_resolve) > 0:
for method in list(to_resolve):
try:
resolver(self, method)
except ValueError:
pass
else:
to_resolve.remove(method)
resolved += 1
if resolved == 0:
raise ValueError("Not all could be resolved")
self.logger.debug('All plugins loaded')
def add_service(self, service):
self.__services[service._config.namespace] = service
def get_service(self, name):
return self.__services[name]
def get_services(self):
return self.__services
def add_schema(self, schema):
if schema.name in self.__schemas:
raise ValueError('Schema "{0}" is already registered'.format(
schema.name
))
self.__schemas[schema.name] = schema
def get_schema(self, name):
return self.__schemas.get(name)
def get_jobs(self):
return self.__jobs
def call_method(self, app, message):
"""Call method from websocket"""
params = message.get('params') or []
service, method_name = message['method'].rsplit('.', 1)
methodobj = getattr(self.get_service(service), method_name)
if not app.authenticated and not hasattr(methodobj, '_no_auth_required'):
app.send_error(message, 'Not authenticated')
return
args = []
if hasattr(methodobj, '_pass_app'):
args.append(app)
# If the method is marked as a @job we need to create a new
# entry to keep track of its state.
job_options = getattr(methodobj, '_job', None)
if job_options:
# Create a job instance with required args
job = Job(message['method'], methodobj, args, job_options)
# Add the job to the queue.
# At this point an `id` is assinged to the job.
self.__jobs.add(job)
else:
job = None
args.extend(params)
if job:
return job.id
else:
return methodobj(*args)
def call(self, method, *params):
service, method = method.rsplit('.', 1)
return getattr(self.get_service(service), method)(*params)
def rollbar_report(self, exc_info):
# Allow rollbar to be disabled via sentinel file or environment var
if (
os.path.exists('/tmp/.rollbar_disabled') or
'ROLLBAR_DISABLED' in os.environ
):
return
extra_data = {}
try:
extra_data['sw_version'] = self.call('system.version')
except:
self.logger.debug('Failed to get system version', exc_info=True)
for path, name in (
('/var/log/middlewared.log', 'middlewared_log'),
):
if os.path.exists(path):
with open(path, 'r') as f:
extra_data[name] = f.read()[-10240:]
rollbar.report_exc_info(exc_info, extra_data=extra_data)
def run(self):
Application.middleware = self
wsserver = WebSocketServer(('127.0.0.1', 6000), Resource(OrderedDict([
('/websocket', Application),
])))
restful_api = RESTfulAPI(self)
apidocs_app.middleware = self
apidocsserver = WSGIServer(('127.0.0.1', 8001), apidocs_app)
restserver = WSGIServer(('127.0.0.1', 8002), restful_api.get_app())
server_threads = [
gevent.spawn(wsserver.serve_forever),
gevent.spawn(apidocsserver.serve_forever),
gevent.spawn(restserver.serve_forever),
gevent.spawn(self.__jobs.run),
]
self.logger.debug('Accepting connections')
gevent.joinall(server_threads)
class Middleware(object):
def __init__(self):
self.logger = logging.getLogger('middleware')
self.__jobs = JobsQueue(self)
self.__schemas = {}
self.__services = {}
self.__wsclients = {}
self.__init_services()
self.__init_rollbar()
self.__plugins_load()
def __init_services(self):
from middlewared.service import CoreService
self.add_service(CoreService(self))
def __init_rollbar(self):
rollbar.init(
'caf06383cba14d5893c4f4d0a40c33a9',
'production' if 'DEVELOPER_MODE' not in os.environ else 'development'
)
def __plugins_load(self):
from middlewared.service import Service, CRUDService, ConfigService
plugins_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'plugins',
)
self.logger.debug('Loading plugins from {0}'.format(plugins_dir))
if not os.path.exists(plugins_dir):
raise ValueError('plugins dir not found')
for f in os.listdir(plugins_dir):
if not f.endswith('.py'):
continue
f = f[:-3]
fp, pathname, description = imp.find_module(f, [plugins_dir])
try:
mod = imp.load_module(f, fp, pathname, description)
finally:
if fp:
fp.close()
for attr in dir(mod):
attr = getattr(mod, attr)
if not inspect.isclass(attr):
continue
if attr in (Service, CRUDService, ConfigService):
continue
if issubclass(attr, Service):
self.add_service(attr(self))
if hasattr(mod, 'setup'):
mod.setup(self)
# Now that all plugins have been loaded we can resolve all method params
# to make sure every schema is patched and references match
from middlewared.schema import resolver # Lazy import so namespace match
to_resolve = []
for service in self.__services.values():
for attr in dir(service):
to_resolve.append(getattr(service, attr))
resolved = 0
while len(to_resolve) > 0:
for method in list(to_resolve):
try:
resolver(self, method)
except ValueError:
pass
else:
to_resolve.remove(method)
resolved += 1
if resolved == 0:
raise ValueError("Not all could be resolved")
self.logger.debug('All plugins loaded')
def register_wsclient(self, client):
self.__wsclients[client.sessionid] = client
def unregister_wsclient(self, client):
self.__wsclients.pop(client.sessionid)
def add_service(self, service):
self.__services[service._config.namespace] = service
def get_service(self, name):
return self.__services[name]
def get_services(self):
return self.__services
def add_schema(self, schema):
if schema.name in self.__schemas:
raise ValueError('Schema "{0}" is already registered'.format(
schema.name
))
self.__schemas[schema.name] = schema
def get_schema(self, name):
return self.__schemas.get(name)
def get_jobs(self):
return self.__jobs
def call_method(self, app, message):
"""Call method from websocket"""
params = message.get('params') or []
service, method_name = message['method'].rsplit('.', 1)
methodobj = getattr(self.get_service(service), method_name)
if not app.authenticated and not hasattr(methodobj, '_no_auth_required'):
app.send_error(message, 'Not authenticated')
return
args = []
if hasattr(methodobj, '_pass_app'):
args.append(app)
# If the method is marked as a @job we need to create a new
# entry to keep track of its state.
job_options = getattr(methodobj, '_job', None)
if job_options:
# Create a job instance with required args
job = Job(self, message['method'], methodobj, args, job_options)
# Add the job to the queue.
# At this point an `id` is assinged to the job.
self.__jobs.add(job)
else:
job = None
args.extend(params)
if job:
return job.id
else:
return methodobj(*args)
def call(self, method, *params):
service, method = method.rsplit('.', 1)
return getattr(self.get_service(service), method)(*params)
def send_event(self, name, event_type, **kwargs):
assert event_type in ('ADDED', 'CHANGED', 'REMOVED')
for sessionid, wsclient in self.__wsclients.iteritems():
try:
wsclient.send_event(name, event_type, **kwargs)
except:
self.logger.warn('Failed to send event {} to {}'.format(name, sessionid), exc_info=True)
def rollbar_report(self, exc_info):
# Allow rollbar to be disabled via sentinel file or environment var
if (
os.path.exists('/tmp/.rollbar_disabled') or
'ROLLBAR_DISABLED' in os.environ
):
return
extra_data = {}
try:
extra_data['sw_version'] = self.call('system.version')
except:
self.logger.debug('Failed to get system version', exc_info=True)
for path, name in (
('/var/log/middlewared.log', 'middlewared_log'),
):
if os.path.exists(path):
with open(path, 'r') as f:
extra_data[name] = f.read()[-10240:]
rollbar.report_exc_info(exc_info, extra_data=extra_data)
def run(self):
Application.middleware = self
wsserver = WebSocketServer(('127.0.0.1', 6000), Resource(OrderedDict([
('/websocket', Application),
])))
restful_api = RESTfulAPI(self)
apidocs_app.middleware = self
apidocsserver = WSGIServer(('127.0.0.1', 8001), apidocs_app)
restserver = WSGIServer(('127.0.0.1', 8002), restful_api.get_app())
server_threads = [
gevent.spawn(wsserver.serve_forever),
gevent.spawn(apidocsserver.serve_forever),
gevent.spawn(restserver.serve_forever),
gevent.spawn(self.__jobs.run),
]
self.logger.debug('Accepting connections')
gevent.joinall(server_threads)