def make_pid_file(self): """ \brief Create a pid file in which we store the PID of the daemon if needed """ if Options().pid_filename and Options().no_daemon == False: atexit.register(self.remove_pid_file) file(Options().pid_filename, "w+").write("%s\n" % str(os.getpid()))
def remove_pid_file(self): """ \brief Remove the pid file (internal usage) """ # The lock file is implicitely released while removing the pid file Log.debug("Removing %s" % Options().pid_filename) if os.path.exists(Options().pid_filename) == True: os.remove(Options().pid_filename)
def _xmlrpc_action(self, action, *args): # The first argument is eventually an authentication token if Options().disable_auth: query, = args else: auth, query = args query['action'] = action if Options().disable_auth: return self.xmlrpc_forward(query) else: return self.xmlrpc_forward(auth, query)
def get_pid_from_pid_file(self): """ \brief Retrieve the PID of the daemon thanks to the pid file. \return None if the pid file is not readable or does not exists """ pid = None if Options().pid_filename: try: f_pid = file(Options().pid_filename, "r") pid = int(f_pid.read().strip()) f_pid.close() except IOError: pid = None return pid
def main(): init_options() Shell.init_options() # This should be taken care of by argparse... if Options().method in [METHOD_APPEND, METHOD_INSERT]: if not Options().target: print "manifold-tables: error: argument -j/--jump is required" sys.exit(0) # Build rule dictionary rule = Rule() rule.object = Options().object fields = [x.strip() for x in Options().fields.split(',')] rule.fields = set(fields) rule.access = Options().access rule.target = Options().target rule_dict = rule.to_dict() shell = Shell() if Options().method == METHOD_APPEND: shell.evaluate("insert into local:policy SET policy_json = '%s'" % json.dumps(rule_dict)) elif Options().method == METHOD_FLUSH: shell.evaluate("delete from local:policy") else: raise Exception, "Not implemented" shell.terminate()
def make_lock_file(self): """ \brief Prepare the lock file required to manage the pid file Initialize Options().lock_file """ if Options().pid_filename and Options().no_daemon == False: Log.debug("Daemonizing using pid file '%s'" % Options().pid_filename) Options().lock_file = lockfile.FileLock(Options().pid_filename) if Options().lock_file.is_locked() == True: log_error("'%s' is already running ('%s' is locked)." % (Options().get_name(), Options().pid_filename)) self.terminate() Options().lock_file.acquire() else: Options().lock_file = None
def xmlrpc_forward(self, request, query, annotations=None): """ """ Log.info("Incoming XMLRPC request, query = %r, annotations = %r" % (self.display_query(query), annotations)) if Options().disable_auth: Log.info("Authentication disabled by configuration") else: if not annotations or not 'authentication' in annotations: msg = "You need to specify an authentication token in annotations" return dict(ResultValue.get_error(ResultValue.FORBIDDEN, msg)) # We expect to find an authentication token in the annotations if annotations: auth = annotations.get('authentication', None) else: auth = {} auth['request'] = request # Check login password try: user = Auth(auth, self.interface).check() except Exception, e: Log.warning( "XMLRPCAPI::xmlrpc_forward: Authentication failed...: %s" % str(e)) msg = "Authentication failed: %s" % e return dict(ResultValue.get_error(ResultValue.FORBIDDEN, msg))
def error(cls, *msg, **ctx): if not Options().log_level in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: return cls.log_message('ERROR', msg, ctx) logger = Log().get_logger() if not Log().get_logger(): traceback.print_exc() sys.exit(0)
def main(): XMLRPCDaemon.init_options() Log.init_options() Daemon.init_options() DBStorage.init_options() Options().parse() XMLRPCDaemon().start()
def log_message(cls, level, msg, ctx): """ \brief Logs an message \param level (string) Log level \param msg (string / list of strings) Message string, or List of message strings \param ctx (dict) Context for the message strings """ caller = None if not Options().log_duplicates: try: count = cls.seen.get(msg, 0) cls.seen[msg] = count + 1 except TypeError as e: # Unhashable types in msg count = 0 if count == 1: msg += ( " -- REPEATED -- Future similar messages will be silently ignored. Please use the --log_duplicates option to allow for duplicates", ) elif count > 1: return if level == 'DEBUG': caller = caller_name(skip=3) # Eventually remove "" added to the configuration file try: paths = tuple( s.strip(' \t\n\r') for s in Options().debug.split(',')) except: paths = None if not paths or not caller.startswith(paths): return logger = Log().get_logger() msg_str = cls.build_message_string(msg, ctx) if logger: logger_fct = getattr(logger, level.lower()) logger_fct("%s(): %s" % (inspect.stack()[2][3], msg_str)) else: cls.print_msg(msg_str, level, caller)
def init_log(self, options=object()): # Initialize self.log (require self.files_to_keep) if self.log: # for debugging by using stdout, log may be equal to None if Options().rsyslog_host: shandler = self.make_handler_rsyslog(Options().rsyslog_host, Options().rsyslog_port, Options().log_level) elif Options().log_file: shandler = self.make_handler_locallog(Options().log_file, Options().log_level)
def log_message(cls, level, msg, ctx): """ \brief Logs an message \param level (string) Log level \param msg (string / list of strings) Message string, or List of message strings \param ctx (dict) Context for the message strings """ caller = None if not Options().log_duplicates: try: count = cls.seen.get(msg, 0) cls.seen[msg] = count + 1 except TypeError, e: # Unhashable types in msg count = 0 if count == 1: msg += (" -- REPEATED -- Future similar messages will be silently ignored. Please use the --log_duplicates option to allow for duplicates",) elif count > 1: return
def start(self): """ \brief Start the daemon """ # Check whether daemon module is properly installed if self.check_python_daemon() == False: self.terminate() import daemon # Prepare Options().lock_file self.make_lock_file() # Prepare the daemon context dcontext = daemon.DaemonContext( detach_process=(not Options().no_daemon), working_directory=Options().working_directory, pidfile=Options().lock_file if not Options().no_daemon else None, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, uid=Options().uid, gid=Options().gid, files_preserve=Log().files_to_keep) # Prepare signal handling to stop properly if the daemon is killed # Note that signal.SIGKILL can't be handled: # http://crunchtools.com/unixlinux-signals-101/ dcontext.signal_map = { signal.SIGTERM: self.signal_handler, signal.SIGQUIT: self.signal_handler, signal.SIGINT: self.signal_handler } if Options().debugmode == True: self.main() else: with dcontext: self.make_pid_file() try: self.main() except Exception, why: Log.error("Unhandled exception in start: %s" % why)
class Daemon(object): __metaclass__ = Singleton DEFAULTS = { # Running "uid": os.getuid(), "gid": os.getgid(), "working_directory": "/", "debugmode": False, "no_daemon": False, "pid_filename": "/var/run/%s.pid" % Options().get_name() } #------------------------------------------------------------------------- # Checks #------------------------------------------------------------------------- def check_python_daemon(self): """ \brief Check whether python-daemon is properly installed \return True if everything is file, False otherwise """ # http://www.python.org/dev/peps/pep-3143/ ret = False try: import daemon getattr(daemon, "DaemonContext") ret = True except AttributeError, e: print e # daemon and python-daemon conflict with each other Log.critical( "Please install python-daemon instead of daemon. Remove daemon first." ) except ImportError: Log.critical( "Please install python-daemon - easy_install python-daemon.")
def get_url(): url = '' options = Options() engine = Options().storage_engine login = Options().storage_login password = Options().storage_password host = Options().storage_host port = Options().storage_port database = Options().storage_database if not database: raise RuntimeError( "check storage_database in /etc/manifold/manifold.conf") if not engine: engine = 'sqlite' if engine == 'sqlite' or engine == 'sqlite3': url = engine + ':///' url += database + '?check_same_thread=False' #URL='sqlite:////var/myslice/db.sqlite?check_same_thread=False' else: url = engine + '://' if not login: raise RuntimeError( "check storage_login in /etc/manifold/manifold.conf") if not password: raise RuntimeError( "check storage_password in /etc/manifold/manifold.conf") if not host: raise RuntimeError( "check storage_host in /etc/manifold/manifold.conf") url += login + ':' + password + '@' + host if port: url += ':' + port url += '/' + database #URL='mysql://*****:*****@localhost/manifold' return url
def critical(cls, *msg, **ctx): if not Options().log_level in ['CRITICAL']: return cls.log_message('CRITICAL', msg, ctx) sys.exit(0)
def init_options(self): """ Prepare options supported by XMLRPCDaemon. """ # Processing opt = Options() opt.add_argument( "-P", "--port", dest = "xmlrpc_port", help = "Port on which the XMLRPC server will listen.", default = 7080 ) # XXX router could be an additional argument opt.add_argument( "-g", "--gateway", dest = "gateway", help = "Gateway exposed by the server, None for acting as a router.", default = None ) opt.add_argument( "-p", "--platform", dest = "platform", help = "Platform exposed by the server, None for acting as a router.", default = None ) opt.add_argument( "-a", "--disable-auth", action="store_true", dest = "disable_auth", help = "Disable authentication", default = False ) opt.add_argument( "-t", "--trusted-roots-path", dest = "trusted_roots_path", help = "Select the directory holding trusted root certificates", default = '/etc/manifold/trusted_roots/' ) opt.add_argument( "-s", "--server-ssl-path", action="store_true", dest = "ssl_path", help = "Select the directory holding the server private key and certificate for SSL", default = '/etc/manifold/keys' )
def init_options(self): opt = Options() opt.add_option("--uid", dest="uid", help="UID used to run the dispatcher.", default=self.DEFAULTS['uid']) opt.add_option("--gid", dest="gid", help="GID used to run the dispatcher.", default=self.DEFAULTS['gid']) opt.add_option("-w", "--working-directory", dest="working_directory", help="Working directory.", default=self.DEFAULTS['working_directory']) opt.add_option("-D", "--debugmode", action="store_false", dest="debugmode", help="Daemon debug mode (useful for developers).", default=self.DEFAULTS['debugmode']) opt.add_option("-n", "--no-daemon", action="store_true", dest="no_daemon", help="Run as daemon (detach from terminal).", default=self.DEFAULTS["no_daemon"]) opt.add_option( "-i", "--pid-file", dest="pid_filename", help="Absolute path to the pid-file to use when running as daemon.", default=self.DEFAULTS['pid_filename'])
def debug(cls, *msg, **ctx): if not Options().log_level in ['DEBUG']: return cls.log_message('DEBUG', msg, ctx)
t = time.time() START = int(t - (t % GRAIN) + GRAIN) END = START + 1800 # 30 min lease if __name__ == '__main__': import sys, inspect if len(sys.argv) > 1 and sys.argv[1] == 'list': print sys.argv[0] for class_name, class_obj in inspect.getmembers( sys.modules[__name__], inspect.isclass): if not class_name.endswith('Tests'): continue print "%s %s" % ( sys.argv[0], class_name, ) for method_name, _ in inspect.getmembers( class_obj, predicate=inspect.ismethod): if not method_name.startswith('test_'): continue print "%s %s.%s" % (sys.argv[0], class_name, method_name) sys.exit(0) Options().log_level = test_options.pop('log_level') unittest.main(**test_options)
def init_options(): opt = Options() opt.add_argument("-Se", "--storage-engine", dest="storage_engine", help="Engine of the local storage", default=None) opt.add_argument("-Sl", "--storage-login", dest="storage_login", help="Login of the local storage", default=None) opt.add_argument("-Sp", "--storage-password", dest="storage_password", help="Password of the local storage", default=None) opt.add_argument("-SH", "--storage-host", dest="storage_host", help="Host of the local storage", default=None) opt.add_argument("-SP", "--storage-port", dest="storage_port", help="Port of the local storage", default=None) opt.add_argument("-Sd", "--storage-database", dest="storage_database", help="Database of the local storage", default=None)
count = cls.seen.get(msg, 0) cls.seen[msg] = count + 1 except TypeError, e: # Unhashable types in msg count = 0 if count == 1: msg += (" -- REPEATED -- Future similar messages will be silently ignored. Please use the --log_duplicates option to allow for duplicates",) elif count > 1: return if level == 'DEBUG': caller = caller_name(skip=3) # Eventually remove "" added to the configuration file try: paths = tuple(s.strip(' \t\n\r') for s in Options().debug.split(',')) except: paths = None if not paths or not caller.startswith(paths): return logger = Log().get_logger() msg_str = cls.build_message_string(msg, ctx) if logger: logger_fct = getattr(logger, level.lower()) logger_fct("%s(): %s" % (inspect.stack()[2][3], msg_str)) else: cls.print_msg(msg_str, level, caller)
import sys, pprint def assert_rv_success(result_value): print type(result_value) if isinstance(result_value, dict): return result_value['value'] assert isinstance(result_value, ResultValue) assert result_value.is_success() records = result_value.get_value() #assert isinstance(records, Records) # ONLY IN ROUTERV2 return records #Options().log_level = 'DEBUG' Options().username = auth['Username'] Options().password = auth['AuthString'] Options().xmlrpc_url = "https://portal.onelab.eu:7080" shell = Shell(interactive=False) shell.select_auth_method('password') command = 'SELECT hrn, hostname FROM resource' result_value = shell.evaluate(command) try: records = assert_rv_success(result_value) print "===== RESULTS =====" print records except Exception, e: print "===== ERROR ====="
def warning(cls, *msg, **ctx): if not Options().log_level in ['DEBUG', 'INFO', 'WARNING']: return cls.log_message('WARNING', msg, ctx)
def info(cls, *msg, **ctx): if not Options().log_level in ['DEBUG', 'INFO']: return cls.log_message('INFO', msg, ctx)
def init_options(self): opt = Options() opt.add_option( "--uid", dest = "uid", help = "UID used to run the dispatcher.", default = self.DEFAULTS['uid'] ) opt.add_option( "--gid", dest = "gid", help = "GID used to run the dispatcher.", default = self.DEFAULTS['gid'] ) opt.add_option( "-w", "--working-directory", dest = "working_directory", help = "Working directory.", default = self.DEFAULTS['working_directory'] ) opt.add_option( "-D", "--debugmode", action = "store_false", dest = "debugmode", help = "Daemon debug mode (useful for developers).", default = self.DEFAULTS['debugmode'] ) opt.add_option( "-n", "--no-daemon", action = "store_true", dest = "no_daemon", help = "Run as daemon (detach from terminal).", default = self.DEFAULTS["no_daemon"] ) opt.add_option( "-i", "--pid-file", dest = "pid_filename", help = "Absolute path to the pid-file to use when running as daemon.", default = self.DEFAULTS['pid_filename'] )
def init_options(self): opt = Options() opt.add_argument( "--rsyslog-enable", action="store_false", dest="rsyslog_enable", help="Specify if log have to be written to a rsyslog server.", default=self.DEFAULTS["rsyslog_enable"]) opt.add_argument("--rsyslog-host", dest="rsyslog_host", help="Rsyslog hostname.", default=self.DEFAULTS["rsyslog_host"]) opt.add_argument("--rsyslog-port", type=int, dest="rsyslog_port", help="Rsyslog port.", default=self.DEFAULTS["rsyslog_port"]) opt.add_argument("-o", "--log-file", dest="log_file", help="Log filename.", default=self.DEFAULTS["log_file"]) opt.add_argument( "-L", "--log-level", dest="log_level", choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], help="Log level", default=self.DEFAULTS["log_level"]) opt.add_argument( "-d", "--debug", dest="debug", help= "Debug paths (a list of coma-separated python path: path.to.module.function).", default=self.DEFAULTS["debug"]) opt.add_argument("--log_duplicates", action="store_true", dest="log_duplicates", help="Remove duplicate messages in logs", default=self.DEFAULTS["log_duplicates"])
# Authors: # Marc-Olivier Buob <*****@*****.**> import sys from manifold.gateways.tdmi import TDMIGateway from manifold.core.query import Query from manifold.core.result_value import ResultValue from manifold.core.router import Router from manifold.util.storage import DBStorage from manifold.util.type import returns, accepts from manifold.util.log import Log from manifold.util.options import Options from manifold.util.predicate import Predicate, eq Options().parse() @accepts(dict) def print_record(record): """ Process a record fetched by the gateway Args: record: A dictionnary storing the fetched record """ if record: print "{" for field_name in sorted(record.keys()): print "\t%s: %s" % (field_name, record[field_name]) print "}" else:
def init_options(): import argparse opt = Options() class InsertAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, 'method', METHOD_INSERT) setattr(namespace, 'index', values[0]) class DeleteAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, 'method', METHOD_DELETE) setattr(namespace, 'index', values[0]) # Method group = opt.add_mutually_exclusive_group(required=True) group.add_argument("-A", "--append", action='store_const', const=METHOD_APPEND, dest='method', help="TODO") group.add_argument("-F", "--flush", action='store_const', const=METHOD_FLUSH, dest='method', help="TODO") group.add_argument("-I", "--insert", action=InsertAction, dest='method', help="TODO") group.add_argument("-D", "--delete", action=DeleteAction, dest='method', help="todo") # Destination opt.add_argument("-o", "--object", dest='object', default=DEFAULT_TABLE, help="TODO. Default is %s" % DEFAULT_TABLE) opt.add_argument("-f", "--fields", dest='fields', default=DEFAULT_FIELDS, help="TODO. Default is %s" % DEFAULT_FIELDS) opt.add_argument("-a", "--access", dest='access', default=DEFAULT_ACTION, help="TODO. Default is %s" % DEFAULT_ACTION) # Target (only required for append or insert opt.add_argument("-j", "--jump", dest='target', help="TODO")
def main(self): """ \brief Runs a XMLRPC server """ Log.info("XMLRPC server daemon (%s) started." % sys.argv[0]) # NOTE it is important to import those files only after daemonization, # since they open files we cannot easily preserve from twisted.web import xmlrpc, server # SSL support from OpenSSL import SSL from twisted.internet import ssl #, reactor #from twisted.internet.protocol import Factory, Protocol #from twisted.internet import reactor # This also imports manifold.util.reactor_thread that uses reactor from manifold.core.router import Router assert not (Options().platform and Options().gateway), "Both gateway and platform cannot be specified at commandline" # This imports twisted code so we need to import it locally from manifold.core.xmlrpc_api import XMLRPCAPI # This should be configurable allowed_capabilities = Capabilities() allowed_capabilities.selection = True allowed_capabilities.projection = True # XXX We should harmonize interfaces between Router and Forwarder if Options().platform: platforms = Storage.execute(Query().get('platform'), format='object') # We pass a single platform to Forwarder platform = [p for p in platforms if p.name == Options().platform][0] self.interface = Forwarder(platform, allowed_capabilities) elif Options().gateway: # XXX user # XXX Change Forwarded initializer #DEPRECATED| platform = Platform(u'dummy', Options().gateway, self.get_gateway_config(Options().gateway), 'user') platform = Platform( platform = u'dummy', gateway_type = Options().gateway, config = self.get_gateway_config(Options().gateway), auth_type = 'user' ) self.interface = Forwarder(platform, allowed_capabilities) else: self.interface = Router() try: def verifyCallback(connection, x509, errnum, errdepth, ok): if not ok: print 'invalid cert from subject:', x509.get_subject() print errnum, errdepth return False else: print "Certs are fine", x509, x509.get_subject() return True ssl_path = Options().ssl_path if not ssl_path or not os.path.exists(ssl_path): print "" print "You need to generate SSL keys and certificate in '%s' to be able to run manifold" % ssl_path print "" print "mkdir -p /etc/manifold/keys" print "openssl genrsa 1024 > /etc/manifold/keys/server.key" print "chmod 400 /etc/manifold/keys/server.key" print "openssl req -new -x509 -nodes -sha1 -days 365 -key /etc/manifold/keys/server.key > /etc/manifold/keys/server.cert" print "" sys.exit(0) server_key_file = "%s/server.key" % ssl_path server_crt_file = "%s/server.cert" % ssl_path Log.tmp("key, cert=", server_key_file, server_crt_file) myContextFactory = ssl.DefaultOpenSSLContextFactory(server_key_file, server_crt_file) ctx = myContextFactory.getContext() ctx.set_verify( SSL.VERIFY_PEER, # | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, verifyCallback ) # Since we have self-signed certs we have to explicitly # tell the server to trust them. #ctx.load_verify_locations("keys/ca.pem") trusted_roots_path = Options().trusted_roots_path if not trusted_roots_path or not os.path.exists(trusted_roots_path): Log.warning("No trusted root found in %s. You won't be able to login using SSL client certificates" % trusted_roots_path) ctx.load_verify_locations(None, ssl_path) #ReactorThread().listenTCP(Options().xmlrpc_port, server.Site(XMLRPCAPI(self.interface, allowNone=True))) ReactorThread().listenSSL(Options().xmlrpc_port, server.Site(XMLRPCAPI(self.interface, allowNone=True)), myContextFactory) ReactorThread().start_reactor() except Exception, e: # TODO If database gets disconnected, we can sleep/attempt reconnection Log.error("Error in XMLRPC API: %s" % str(e))
#!/usr/bin/env python #! -*- coding: utf-8 -*- import sys, pprint from manifold.auth import * from manifold.core.router import Router from manifold.core.query import Query from manifold.util.log import Log from manifold.util.options import Options from config import auth Log.init_options() Options().parse() try: print "**", Options(), "**" except Exception, e: log_error(str(e)) DEFAULT_SLICE = 'ple.upmc.myslicedemo' def print_err(err): print '-' * 80 print 'Exception', err['code'], 'raised by', err['origin'], ':', err[ 'description'] for line in err['traceback'].split("\n"): print "\t", line print ''
def init_options(self): opt = Options() opt.add_option( "--rsyslog-enable", action = "store_false", dest = "rsyslog_enable", help = "Specify if log have to be written to a rsyslog server.", default = self.DEFAULTS["rsyslog_enable"] ) opt.add_option( "--rsyslog-host", dest = "rsyslog_host", help = "Rsyslog hostname.", default = self.DEFAULTS["rsyslog_host"] ) opt.add_option( "--rsyslog-port", type = "int", dest = "rsyslog_port", help = "Rsyslog port.", default = self.DEFAULTS["rsyslog_port"] ) opt.add_option( "-o", "--log-file", dest = "log_file", help = "Log filename.", default = self.DEFAULTS["log_file"] ) opt.add_option( "-L", "--log-level", dest = "log_level", choices = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], help = "Log level", default = self.DEFAULTS["log_level"] ) opt.add_option( "-d", "--debug", dest = "debug", help = "Debug paths (a list of coma-separated python path: path.to.module.function).", default = self.DEFAULTS["debug"] ) opt.add_option( "", "--log_duplicates", action = "store_true", dest = "log_duplicates", help = "Remove duplicate messages in logs", default = self.DEFAULTS["log_duplicates"] )