def make_pid_file(self):
"""
\brief Create a pid file in which we store the PID of the daemon if needed
"""
if Options().pid_filename and Options().no_daemon == False:
atexit.register(self.remove_pid_file)
file(Options().pid_filename, "w+").write("%s\n" % str(os.getpid()))
def remove_pid_file(self):
"""
\brief Remove the pid file (internal usage)
"""
# The lock file is implicitely released while removing the pid file
Log.debug("Removing %s" % Options().pid_filename)
if os.path.exists(Options().pid_filename) == True:
os.remove(Options().pid_filename)
def _xmlrpc_action(self, action, *args):
# The first argument is eventually an authentication token
if Options().disable_auth:
query, = args
else:
auth, query = args
query['action'] = action
if Options().disable_auth:
return self.xmlrpc_forward(query)
else:
return self.xmlrpc_forward(auth, query)
def get_pid_from_pid_file(self):
"""
\brief Retrieve the PID of the daemon thanks to the pid file.
\return None if the pid file is not readable or does not exists
"""
pid = None
if Options().pid_filename:
try:
f_pid = file(Options().pid_filename, "r")
pid = int(f_pid.read().strip())
f_pid.close()
except IOError:
pid = None
return pid
def main():
init_options()
Shell.init_options()
# This should be taken care of by argparse...
if Options().method in [METHOD_APPEND, METHOD_INSERT]:
if not Options().target:
print "manifold-tables: error: argument -j/--jump is required"
sys.exit(0)
# Build rule dictionary
rule = Rule()
rule.object = Options().object
fields = [x.strip() for x in Options().fields.split(',')]
rule.fields = set(fields)
rule.access = Options().access
rule.target = Options().target
rule_dict = rule.to_dict()
shell = Shell()
if Options().method == METHOD_APPEND:
shell.evaluate("insert into local:policy SET policy_json = '%s'" %
json.dumps(rule_dict))
elif Options().method == METHOD_FLUSH:
shell.evaluate("delete from local:policy")
else:
raise Exception, "Not implemented"
shell.terminate()
def make_lock_file(self):
"""
\brief Prepare the lock file required to manage the pid file
Initialize Options().lock_file
"""
if Options().pid_filename and Options().no_daemon == False:
Log.debug("Daemonizing using pid file '%s'" %
Options().pid_filename)
Options().lock_file = lockfile.FileLock(Options().pid_filename)
if Options().lock_file.is_locked() == True:
log_error("'%s' is already running ('%s' is locked)." %
(Options().get_name(), Options().pid_filename))
self.terminate()
Options().lock_file.acquire()
else:
Options().lock_file = None
def xmlrpc_forward(self, request, query, annotations=None):
"""
"""
Log.info("Incoming XMLRPC request, query = %r, annotations = %r" %
(self.display_query(query), annotations))
if Options().disable_auth:
Log.info("Authentication disabled by configuration")
else:
if not annotations or not 'authentication' in annotations:
msg = "You need to specify an authentication token in annotations"
return dict(ResultValue.get_error(ResultValue.FORBIDDEN, msg))
# We expect to find an authentication token in the annotations
if annotations:
auth = annotations.get('authentication', None)
else:
auth = {}
auth['request'] = request
# Check login password
try:
user = Auth(auth, self.interface).check()
except Exception, e:
Log.warning(
"XMLRPCAPI::xmlrpc_forward: Authentication failed...: %s" %
str(e))
msg = "Authentication failed: %s" % e
return dict(ResultValue.get_error(ResultValue.FORBIDDEN, msg))
def error(cls, *msg, **ctx):
if not Options().log_level in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
return
cls.log_message('ERROR', msg, ctx)
logger = Log().get_logger()
if not Log().get_logger():
traceback.print_exc()
sys.exit(0)
def main():
XMLRPCDaemon.init_options()
Log.init_options()
Daemon.init_options()
DBStorage.init_options()
Options().parse()
XMLRPCDaemon().start()
def log_message(cls, level, msg, ctx):
"""
\brief Logs an message
\param level (string) Log level
\param msg (string / list of strings) Message string, or List of message strings
\param ctx (dict) Context for the message strings
"""
caller = None
if not Options().log_duplicates:
try:
count = cls.seen.get(msg, 0)
cls.seen[msg] = count + 1
except TypeError as e:
# Unhashable types in msg
count = 0
if count == 1:
msg += (
" -- REPEATED -- Future similar messages will be silently ignored. Please use the --log_duplicates option to allow for duplicates",
)
elif count > 1:
return
if level == 'DEBUG':
caller = caller_name(skip=3)
# Eventually remove "" added to the configuration file
try:
paths = tuple(
s.strip(' \t\n\r') for s in Options().debug.split(','))
except:
paths = None
if not paths or not caller.startswith(paths):
return
logger = Log().get_logger()
msg_str = cls.build_message_string(msg, ctx)
if logger:
logger_fct = getattr(logger, level.lower())
logger_fct("%s(): %s" % (inspect.stack()[2][3], msg_str))
else:
cls.print_msg(msg_str, level, caller)
def init_log(self, options=object()):
# Initialize self.log (require self.files_to_keep)
if self.log: # for debugging by using stdout, log may be equal to None
if Options().rsyslog_host:
shandler = self.make_handler_rsyslog(Options().rsyslog_host,
Options().rsyslog_port,
Options().log_level)
elif Options().log_file:
shandler = self.make_handler_locallog(Options().log_file,
Options().log_level)
def log_message(cls, level, msg, ctx):
"""
\brief Logs an message
\param level (string) Log level
\param msg (string / list of strings) Message string, or List of message strings
\param ctx (dict) Context for the message strings
"""
caller = None
if not Options().log_duplicates:
try:
count = cls.seen.get(msg, 0)
cls.seen[msg] = count + 1
except TypeError, e:
# Unhashable types in msg
count = 0
if count == 1:
msg += (" -- REPEATED -- Future similar messages will be silently ignored. Please use the --log_duplicates option to allow for duplicates",)
elif count > 1:
return
def start(self):
"""
\brief Start the daemon
"""
# Check whether daemon module is properly installed
if self.check_python_daemon() == False:
self.terminate()
import daemon
# Prepare Options().lock_file
self.make_lock_file()
# Prepare the daemon context
dcontext = daemon.DaemonContext(
detach_process=(not Options().no_daemon),
working_directory=Options().working_directory,
pidfile=Options().lock_file if not Options().no_daemon else None,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
uid=Options().uid,
gid=Options().gid,
files_preserve=Log().files_to_keep)
# Prepare signal handling to stop properly if the daemon is killed
# Note that signal.SIGKILL can't be handled:
# http://crunchtools.com/unixlinux-signals-101/
dcontext.signal_map = {
signal.SIGTERM: self.signal_handler,
signal.SIGQUIT: self.signal_handler,
signal.SIGINT: self.signal_handler
}
if Options().debugmode == True:
self.main()
else:
with dcontext:
self.make_pid_file()
try:
self.main()
except Exception, why:
Log.error("Unhandled exception in start: %s" % why)
class Daemon(object):
__metaclass__ = Singleton
DEFAULTS = {
# Running
"uid": os.getuid(),
"gid": os.getgid(),
"working_directory": "/",
"debugmode": False,
"no_daemon": False,
"pid_filename": "/var/run/%s.pid" % Options().get_name()
}
#-------------------------------------------------------------------------
# Checks
#-------------------------------------------------------------------------
def check_python_daemon(self):
"""
\brief Check whether python-daemon is properly installed
\return True if everything is file, False otherwise
"""
# http://www.python.org/dev/peps/pep-3143/
ret = False
try:
import daemon
getattr(daemon, "DaemonContext")
ret = True
except AttributeError, e:
print e
# daemon and python-daemon conflict with each other
Log.critical(
"Please install python-daemon instead of daemon. Remove daemon first."
)
except ImportError:
Log.critical(
"Please install python-daemon - easy_install python-daemon.")
def get_url():
url = ''
options = Options()
engine = Options().storage_engine
login = Options().storage_login
password = Options().storage_password
host = Options().storage_host
port = Options().storage_port
database = Options().storage_database
if not database:
raise RuntimeError(
"check storage_database in /etc/manifold/manifold.conf")
if not engine:
engine = 'sqlite'
if engine == 'sqlite' or engine == 'sqlite3':
url = engine + ':///'
url += database + '?check_same_thread=False'
#URL='sqlite:////var/myslice/db.sqlite?check_same_thread=False'
else:
url = engine + '://'
if not login:
raise RuntimeError(
"check storage_login in /etc/manifold/manifold.conf")
if not password:
raise RuntimeError(
"check storage_password in /etc/manifold/manifold.conf")
if not host:
raise RuntimeError(
"check storage_host in /etc/manifold/manifold.conf")
url += login + ':' + password + '@' + host
if port:
url += ':' + port
url += '/' + database
#URL='mysql://*****:*****@localhost/manifold'
return url
def critical(cls, *msg, **ctx):
if not Options().log_level in ['CRITICAL']:
return
cls.log_message('CRITICAL', msg, ctx)
sys.exit(0)
def init_options(self):
"""
Prepare options supported by XMLRPCDaemon.
"""
# Processing
opt = Options()
opt.add_argument(
"-P", "--port", dest = "xmlrpc_port",
help = "Port on which the XMLRPC server will listen.",
default = 7080
)
# XXX router could be an additional argument
opt.add_argument(
"-g", "--gateway", dest = "gateway",
help = "Gateway exposed by the server, None for acting as a router.",
default = None
)
opt.add_argument(
"-p", "--platform", dest = "platform",
help = "Platform exposed by the server, None for acting as a router.",
default = None
)
opt.add_argument(
"-a", "--disable-auth", action="store_true", dest = "disable_auth",
help = "Disable authentication",
default = False
)
opt.add_argument(
"-t", "--trusted-roots-path", dest = "trusted_roots_path",
help = "Select the directory holding trusted root certificates",
default = '/etc/manifold/trusted_roots/'
)
opt.add_argument(
"-s", "--server-ssl-path", action="store_true", dest = "ssl_path",
help = "Select the directory holding the server private key and certificate for SSL",
default = '/etc/manifold/keys'
)
def init_options(self):
opt = Options()
opt.add_option("--uid",
dest="uid",
help="UID used to run the dispatcher.",
default=self.DEFAULTS['uid'])
opt.add_option("--gid",
dest="gid",
help="GID used to run the dispatcher.",
default=self.DEFAULTS['gid'])
opt.add_option("-w",
"--working-directory",
dest="working_directory",
help="Working directory.",
default=self.DEFAULTS['working_directory'])
opt.add_option("-D",
"--debugmode",
action="store_false",
dest="debugmode",
help="Daemon debug mode (useful for developers).",
default=self.DEFAULTS['debugmode'])
opt.add_option("-n",
"--no-daemon",
action="store_true",
dest="no_daemon",
help="Run as daemon (detach from terminal).",
default=self.DEFAULTS["no_daemon"])
opt.add_option(
"-i",
"--pid-file",
dest="pid_filename",
help="Absolute path to the pid-file to use when running as daemon.",
default=self.DEFAULTS['pid_filename'])
def debug(cls, *msg, **ctx):
if not Options().log_level in ['DEBUG']:
return
cls.log_message('DEBUG', msg, ctx)
t = time.time()
START = int(t - (t % GRAIN) + GRAIN)
END = START + 1800 # 30 min lease
if __name__ == '__main__':
import sys, inspect
if len(sys.argv) > 1 and sys.argv[1] == 'list':
print sys.argv[0]
for class_name, class_obj in inspect.getmembers(
sys.modules[__name__], inspect.isclass):
if not class_name.endswith('Tests'):
continue
print "%s %s" % (
sys.argv[0],
class_name,
)
for method_name, _ in inspect.getmembers(
class_obj, predicate=inspect.ismethod):
if not method_name.startswith('test_'):
continue
print "%s %s.%s" % (sys.argv[0], class_name, method_name)
sys.exit(0)
Options().log_level = test_options.pop('log_level')
unittest.main(**test_options)
def init_options():
opt = Options()
opt.add_argument("-Se",
"--storage-engine",
dest="storage_engine",
help="Engine of the local storage",
default=None)
opt.add_argument("-Sl",
"--storage-login",
dest="storage_login",
help="Login of the local storage",
default=None)
opt.add_argument("-Sp",
"--storage-password",
dest="storage_password",
help="Password of the local storage",
default=None)
opt.add_argument("-SH",
"--storage-host",
dest="storage_host",
help="Host of the local storage",
default=None)
opt.add_argument("-SP",
"--storage-port",
dest="storage_port",
help="Port of the local storage",
default=None)
opt.add_argument("-Sd",
"--storage-database",
dest="storage_database",
help="Database of the local storage",
default=None)
count = cls.seen.get(msg, 0)
cls.seen[msg] = count + 1
except TypeError, e:
# Unhashable types in msg
count = 0
if count == 1:
msg += (" -- REPEATED -- Future similar messages will be silently ignored. Please use the --log_duplicates option to allow for duplicates",)
elif count > 1:
return
if level == 'DEBUG':
caller = caller_name(skip=3)
# Eventually remove "" added to the configuration file
try:
paths = tuple(s.strip(' \t\n\r') for s in Options().debug.split(','))
except:
paths = None
if not paths or not caller.startswith(paths):
return
logger = Log().get_logger()
msg_str = cls.build_message_string(msg, ctx)
if logger:
logger_fct = getattr(logger, level.lower())
logger_fct("%s(): %s" % (inspect.stack()[2][3], msg_str))
else:
cls.print_msg(msg_str, level, caller)
import sys, pprint
def assert_rv_success(result_value):
print type(result_value)
if isinstance(result_value, dict):
return result_value['value']
assert isinstance(result_value, ResultValue)
assert result_value.is_success()
records = result_value.get_value()
#assert isinstance(records, Records) # ONLY IN ROUTERV2
return records
#Options().log_level = 'DEBUG'
Options().username = auth['Username']
Options().password = auth['AuthString']
Options().xmlrpc_url = "https://portal.onelab.eu:7080"
shell = Shell(interactive=False)
shell.select_auth_method('password')
command = 'SELECT hrn, hostname FROM resource'
result_value = shell.evaluate(command)
try:
records = assert_rv_success(result_value)
print "===== RESULTS ====="
print records
except Exception, e:
print "===== ERROR ====="
def warning(cls, *msg, **ctx):
if not Options().log_level in ['DEBUG', 'INFO', 'WARNING']:
return
cls.log_message('WARNING', msg, ctx)
def info(cls, *msg, **ctx):
if not Options().log_level in ['DEBUG', 'INFO']:
return
cls.log_message('INFO', msg, ctx)
def init_options(self):
opt = Options()
opt.add_option(
"--uid", dest = "uid",
help = "UID used to run the dispatcher.",
default = self.DEFAULTS['uid']
)
opt.add_option(
"--gid", dest = "gid",
help = "GID used to run the dispatcher.",
default = self.DEFAULTS['gid']
)
opt.add_option(
"-w", "--working-directory", dest = "working_directory",
help = "Working directory.",
default = self.DEFAULTS['working_directory']
)
opt.add_option(
"-D", "--debugmode", action = "store_false", dest = "debugmode",
help = "Daemon debug mode (useful for developers).",
default = self.DEFAULTS['debugmode']
)
opt.add_option(
"-n", "--no-daemon", action = "store_true", dest = "no_daemon",
help = "Run as daemon (detach from terminal).",
default = self.DEFAULTS["no_daemon"]
)
opt.add_option(
"-i", "--pid-file", dest = "pid_filename",
help = "Absolute path to the pid-file to use when running as daemon.",
default = self.DEFAULTS['pid_filename']
)
def init_options(self):
opt = Options()
opt.add_argument(
"--rsyslog-enable",
action="store_false",
dest="rsyslog_enable",
help="Specify if log have to be written to a rsyslog server.",
default=self.DEFAULTS["rsyslog_enable"])
opt.add_argument("--rsyslog-host",
dest="rsyslog_host",
help="Rsyslog hostname.",
default=self.DEFAULTS["rsyslog_host"])
opt.add_argument("--rsyslog-port",
type=int,
dest="rsyslog_port",
help="Rsyslog port.",
default=self.DEFAULTS["rsyslog_port"])
opt.add_argument("-o",
"--log-file",
dest="log_file",
help="Log filename.",
default=self.DEFAULTS["log_file"])
opt.add_argument(
"-L",
"--log-level",
dest="log_level",
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
help="Log level",
default=self.DEFAULTS["log_level"])
opt.add_argument(
"-d",
"--debug",
dest="debug",
help=
"Debug paths (a list of coma-separated python path: path.to.module.function).",
default=self.DEFAULTS["debug"])
opt.add_argument("--log_duplicates",
action="store_true",
dest="log_duplicates",
help="Remove duplicate messages in logs",
default=self.DEFAULTS["log_duplicates"])
# Authors:
# Marc-Olivier Buob <*****@*****.**>
import sys
from manifold.gateways.tdmi import TDMIGateway
from manifold.core.query import Query
from manifold.core.result_value import ResultValue
from manifold.core.router import Router
from manifold.util.storage import DBStorage
from manifold.util.type import returns, accepts
from manifold.util.log import Log
from manifold.util.options import Options
from manifold.util.predicate import Predicate, eq
Options().parse()
@accepts(dict)
def print_record(record):
"""
Process a record fetched by the gateway
Args:
record: A dictionnary storing the fetched record
"""
if record:
print "{"
for field_name in sorted(record.keys()):
print "\t%s: %s" % (field_name, record[field_name])
print "}"
else:
def init_options():
import argparse
opt = Options()
class InsertAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, 'method', METHOD_INSERT)
setattr(namespace, 'index', values[0])
class DeleteAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, 'method', METHOD_DELETE)
setattr(namespace, 'index', values[0])
# Method
group = opt.add_mutually_exclusive_group(required=True)
group.add_argument("-A",
"--append",
action='store_const',
const=METHOD_APPEND,
dest='method',
help="TODO")
group.add_argument("-F",
"--flush",
action='store_const',
const=METHOD_FLUSH,
dest='method',
help="TODO")
group.add_argument("-I",
"--insert",
action=InsertAction,
dest='method',
help="TODO")
group.add_argument("-D",
"--delete",
action=DeleteAction,
dest='method',
help="todo")
# Destination
opt.add_argument("-o",
"--object",
dest='object',
default=DEFAULT_TABLE,
help="TODO. Default is %s" % DEFAULT_TABLE)
opt.add_argument("-f",
"--fields",
dest='fields',
default=DEFAULT_FIELDS,
help="TODO. Default is %s" % DEFAULT_FIELDS)
opt.add_argument("-a",
"--access",
dest='access',
default=DEFAULT_ACTION,
help="TODO. Default is %s" % DEFAULT_ACTION)
# Target (only required for append or insert
opt.add_argument("-j", "--jump", dest='target', help="TODO")
def main(self):
"""
\brief Runs a XMLRPC server
"""
Log.info("XMLRPC server daemon (%s) started." % sys.argv[0])
# NOTE it is important to import those files only after daemonization,
# since they open files we cannot easily preserve
from twisted.web import xmlrpc, server
# SSL support
from OpenSSL import SSL
from twisted.internet import ssl #, reactor
#from twisted.internet.protocol import Factory, Protocol
#from twisted.internet import reactor
# This also imports manifold.util.reactor_thread that uses reactor
from manifold.core.router import Router
assert not (Options().platform and Options().gateway), "Both gateway and platform cannot be specified at commandline"
# This imports twisted code so we need to import it locally
from manifold.core.xmlrpc_api import XMLRPCAPI
# This should be configurable
allowed_capabilities = Capabilities()
allowed_capabilities.selection = True
allowed_capabilities.projection = True
# XXX We should harmonize interfaces between Router and Forwarder
if Options().platform:
platforms = Storage.execute(Query().get('platform'), format='object')
# We pass a single platform to Forwarder
platform = [p for p in platforms if p.name == Options().platform][0]
self.interface = Forwarder(platform, allowed_capabilities)
elif Options().gateway:
# XXX user
# XXX Change Forwarded initializer
#DEPRECATED| platform = Platform(u'dummy', Options().gateway, self.get_gateway_config(Options().gateway), 'user')
platform = Platform(
platform = u'dummy',
gateway_type = Options().gateway,
config = self.get_gateway_config(Options().gateway),
auth_type = 'user'
)
self.interface = Forwarder(platform, allowed_capabilities)
else:
self.interface = Router()
try:
def verifyCallback(connection, x509, errnum, errdepth, ok):
if not ok:
print 'invalid cert from subject:', x509.get_subject()
print errnum, errdepth
return False
else:
print "Certs are fine", x509, x509.get_subject()
return True
ssl_path = Options().ssl_path
if not ssl_path or not os.path.exists(ssl_path):
print ""
print "You need to generate SSL keys and certificate in '%s' to be able to run manifold" % ssl_path
print ""
print "mkdir -p /etc/manifold/keys"
print "openssl genrsa 1024 > /etc/manifold/keys/server.key"
print "chmod 400 /etc/manifold/keys/server.key"
print "openssl req -new -x509 -nodes -sha1 -days 365 -key /etc/manifold/keys/server.key > /etc/manifold/keys/server.cert"
print ""
sys.exit(0)
server_key_file = "%s/server.key" % ssl_path
server_crt_file = "%s/server.cert" % ssl_path
Log.tmp("key, cert=", server_key_file, server_crt_file)
myContextFactory = ssl.DefaultOpenSSLContextFactory(server_key_file, server_crt_file)
ctx = myContextFactory.getContext()
ctx.set_verify(
SSL.VERIFY_PEER, # | SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
verifyCallback
)
# Since we have self-signed certs we have to explicitly
# tell the server to trust them.
#ctx.load_verify_locations("keys/ca.pem")
trusted_roots_path = Options().trusted_roots_path
if not trusted_roots_path or not os.path.exists(trusted_roots_path):
Log.warning("No trusted root found in %s. You won't be able to login using SSL client certificates" % trusted_roots_path)
ctx.load_verify_locations(None, ssl_path)
#ReactorThread().listenTCP(Options().xmlrpc_port, server.Site(XMLRPCAPI(self.interface, allowNone=True)))
ReactorThread().listenSSL(Options().xmlrpc_port, server.Site(XMLRPCAPI(self.interface, allowNone=True)), myContextFactory)
ReactorThread().start_reactor()
except Exception, e:
# TODO If database gets disconnected, we can sleep/attempt reconnection
Log.error("Error in XMLRPC API: %s" % str(e))
#!/usr/bin/env python
#! -*- coding: utf-8 -*-
import sys, pprint
from manifold.auth import *
from manifold.core.router import Router
from manifold.core.query import Query
from manifold.util.log import Log
from manifold.util.options import Options
from config import auth
Log.init_options()
Options().parse()
try:
print "**", Options(), "**"
except Exception, e:
log_error(str(e))
DEFAULT_SLICE = 'ple.upmc.myslicedemo'
def print_err(err):
print '-' * 80
print 'Exception', err['code'], 'raised by', err['origin'], ':', err[
'description']
for line in err['traceback'].split("\n"):
print "\t", line
print ''
def init_options(self):
opt = Options()
opt.add_option(
"--rsyslog-enable", action = "store_false", dest = "rsyslog_enable",
help = "Specify if log have to be written to a rsyslog server.",
default = self.DEFAULTS["rsyslog_enable"]
)
opt.add_option(
"--rsyslog-host", dest = "rsyslog_host",
help = "Rsyslog hostname.",
default = self.DEFAULTS["rsyslog_host"]
)
opt.add_option(
"--rsyslog-port", type = "int", dest = "rsyslog_port",
help = "Rsyslog port.",
default = self.DEFAULTS["rsyslog_port"]
)
opt.add_option(
"-o", "--log-file", dest = "log_file",
help = "Log filename.",
default = self.DEFAULTS["log_file"]
)
opt.add_option(
"-L", "--log-level", dest = "log_level",
choices = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
help = "Log level",
default = self.DEFAULTS["log_level"]
)
opt.add_option(
"-d", "--debug", dest = "debug",
help = "Debug paths (a list of coma-separated python path: path.to.module.function).",
default = self.DEFAULTS["debug"]
)
opt.add_option(
"", "--log_duplicates", action = "store_true", dest = "log_duplicates",
help = "Remove duplicate messages in logs",
default = self.DEFAULTS["log_duplicates"]
)
