def set(self, key, value):
# extend set so we can log mood changes
if key == 'mood':
log.info('componentstate', 'mood of %s changed to %s',
self.get('name'), moods.get(value).name)
flavors.StateCacheable.set(self, key, value)
if key == 'mood' and value == self.get('moodPending'):
# we have reached our pending mood
self.set('moodPending', None)
def _readConfig(workerFile, options):
# modifies options dict in-place
log.info('worker', 'Reading configuration from %s' % workerFile)
try:
cfg = config.WorkerConfigXML(workerFile)
except config.ConfigError, value:
raise errors.FatalError(
"Could not load configuration from %s: %s" % (
workerFile, value))
def _readConfig(confXml, options):
# modifies options dict in-place
log.info('rrdmon', 'Reading configuration from %s' % confXml)
cfg = config.ConfigParser(confXml).parse()
# command-line debug > environment debug > config file debug
if not options.debug and cfg['debug'] \
and not 'FLU_DEBUG' in os.environ:
options.debug = cfg['debug']
return cfg
def registerVFSJelly():
"""Register the jelly used by different backends
"""
from flumotion.common.vfsgnome import registerGnomeVFSJelly
registerGnomeVFSJelly()
from flumotion.common.vfsgio import registerGIOJelly
registerGIOJelly()
log.info('jelly', 'VFS registered')
def catching(proc, *args, **kwargs):
env = os.getenv('FLU_ATEXIT').split(' ')
fqfn = env.pop(0)
log.info('atexit', 'FLU_ATEXIT set, will call %s(*%r) on exit',
fqfn, env)
atexitproc = reflect.namedAny(fqfn)
try:
return proc(*args, **kwargs)
finally:
log.info('atexit', 'trying to call %r(*%r)',
atexitproc, env)
atexitproc(*env)
def cleanup_reactor(force=False):
log.debug('comptest', 'running cleanup_reactor...')
delayed = reactor.getDelayedCalls()
for dc in delayed:
dc.cancel()
# the rest is taken from twisted trial...
sels = reactor.removeAll()
if sels:
log.info('comptest', 'leftover selectables...: %r %r' %
(sels, reactor.waker))
for sel in sels:
if interfaces.IProcessTransport.providedBy(sel):
sel.signalProcess('KILL')
def tryV4L2():
log.debug('webcam', 'trying v4l2')
version = gstreamer.get_plugin_version('video4linux2')
minVersion = (0, 10, 5, 1)
if not version or version < minVersion:
log.info('webcam', 'v4l2 version %r too old (need >=%r)',
version, minVersion)
return defer.fail(NotImplementedError())
pipeline = 'v4l2src name=source device=%s ! fakesink' % (device, )
d = do_element_check(pipeline, 'source', probeDevice,
state=gst.STATE_PAUSED, set_state_deferred=True)
return d
def listDirectory(path):
"""List the directory called path
@returns: the directory
@rtype: deferred that will fire an object implementing L{IDirectory}
"""
global _backends
if not _backends:
_registerBackends()
if not _backends:
raise AssertionError(
"there are no vfs backends available")
backend = _backends[0]
log.info('vfs', 'listing directory %s using %r' % (path, backend))
directory = backend(path)
directory.cacheFiles()
return succeed(directory)
def main(args):
parser = OptionParser(domain="flumotion-job")
log.debug('job', 'Parsing arguments (%r)' % ', '.join(args))
options, args = parser.parse_args(args)
# check if a config file was specified; if so, parse config and copy over
if len(args) != 3:
parser.error("must pass an avatarId and a path to the socket: %r" %
args)
avatarId = args[1]
socket = args[2]
# log our standardized starting marker
log.info('job', "Starting job '%s'" % avatarId)
# register all package paths (FIXME: this should go away when
# components and all deps come from manager)
# this is still necessary so that code from other projects can be imported
from flumotion.common import setup
setup.setupPackagePath()
log.info('job', 'Connecting to worker on socket %s' % (socket))
job_factory = job.JobClientFactory(avatarId)
reactor.connectWith(fdserver.FDConnector, socket, job_factory,
10, checkPID=False)
# should probably move this to boot
if 'FLU_PROFILE' in os.environ:
try:
import statprof
statprof.start()
print 'Profiling started.'
def stop_profiling():
statprof.stop()
statprof.display()
reactor.addSystemEventTrigger('before', 'shutdown',
stop_profiling)
except ImportError, e:
print ('Profiling requested, but statprof is not available (%s)'
% e)
def main(args):
parser = options.OptionParser(domain="flumotion-job")
log.debug('job', 'Parsing arguments (%r)' % ', '.join(args))
opts, args = parser.parse_args(args)
# check if a config file was specified; if so, parse config and copy over
if len(args) != 3:
parser.error("must pass an avatarId and a path to the socket: %r" %
args)
avatarId = args[1]
socket = args[2]
# log our standardized starting marker
log.info('job', "Starting job '%s'" % avatarId)
# register all package paths (FIXME: this should go away when
# components and all deps come from manager)
# this is still necessary so that code from other projects can be imported
from flumotion.common import setup
setup.setupPackagePath()
log.info('job', 'Connecting to worker on socket %s' % (socket))
job_factory = job.JobClientFactory(avatarId)
c = fdserver.FDConnector(socket, job_factory, 10, checkPID=False,
reactor=reactor)
c.connect()
reactor.addSystemEventTrigger('before', 'shutdown',
job_factory.medium.shutdownHandler)
# log our standardized started marker
log.info('job', "Started job '%s'" % avatarId)
reactor.run()
# log our standardized stopping marker
log.info('job', "Stopping job '%s'" % avatarId)
# log our standardized stopped marker
log.info('job', "Stopped job '%s'" % avatarId)
return 0
def _registerBackends():
global _backends
for backend, attributeName in [
('flumotion.common.vfsgio', 'GIODirectory'),
('flumotion.common.vfsgnome', 'GnomeVFSDirectory'),
]:
try:
module = __import__(backend, {}, {}, ' ')
except ImportError:
log.info('vfs', 'skipping backend %s, dependency missing' % (
backend, ))
continue
log.info('vfs', 'adding backend %s' % (backend, ))
backend = getattr(module, attributeName)
try:
backend('/')
except ImportError:
continue
_backends.append(backend)
registerVFSJelly()
def startup(processType, processName, daemonize=False, daemonizeTo='/'):
"""
Prepare a process for starting, logging appropriate standarised messages.
First daemonizes the process, if daemonize is true.
@param processType: The process type, for example 'worker'. Used
as the first part of the log file and PID file names.
@type processType: str
@param processName: The service name of the process. Used to
disambiguate different instances of the same daemon.
Used as the second part of log file and PID file names.
@type processName: str
@param daemonize: whether to daemonize the current process.
@type daemonize: bool
@param daemonizeTo: The directory that the daemon should run in.
@type daemonizeTo: str
"""
log.info(processType, "Starting %s '%s'", processType, processName)
if daemonize:
_daemonizeHelper(processType, daemonizeTo, processName)
log.info(processType, "Started %s '%s'", processType, processName)
def shutdownStarted():
log.info(processType, "Stopping %s '%s'", processType, processName)
def shutdownEnded():
log.info(processType, "Stopped %s '%s'", processType, processName)
# import inside function so we avoid affecting startup
from twisted.internet import reactor
reactor.addSystemEventTrigger('before', 'shutdown',
shutdownStarted)
reactor.addSystemEventTrigger('after', 'shutdown',
shutdownEnded)
def listDirectory(path):
"""List the directory called path
Raises L{flumotion.common.errors.NotDirectoryError} if directoryName is
not a directory.
@param path: the name of the directory to list
@type path: string
@returns: the directory
@rtype: deferred that will fire an object implementing L{IDirectory}
"""
global _backends
if not _backends:
_registerBackends()
if not _backends:
raise AssertionError(
"there are no vfs backends available")
backend = _backends[0]
log.info('vfs', 'listing directory %s using %r' % (path, backend))
try:
directory = backend(path)
directory.cacheFiles()
return succeed(directory)
except Exception, e:
return fail(e)
def registerGnomeVFSJelly():
"""Register the jelly used by the GnomeVFS VFS backend.
"""
setUnjellyableForClass(GnomeVFSFile, GnomeVFSFile)
setUnjellyableForClass(GnomeVFSDirectory, GnomeVFSDirectory)
log.info('jelly', 'GnomeVFS registered')
def registerGIOJelly():
"""Register the jelly used by the GIO VFS backend.
"""
setUnjellyableForClass(GIOFile, GIOFile)
setUnjellyableForClass(GIODirectory, GIODirectory)
log.info('jelly', 'GIO registered')
options.name = "unnamed"
log.debug("manager", "Setting name to unnamed")
else:
options.name = name
log.debug("manager", "Setting name to %s based on path" % name)
# check for wrong options/arguments
if not options.transport in ["ssl", "tcp"]:
sys.stderr.write("ERROR: wrong transport %s, must be ssl or tcp\n" % options.transport)
return 1
# register package path
setup.setupPackagePath()
# log our standardized starting marker
log.info("manager", "Starting manager '%s'" % options.name)
log.debug("manager", "Running Flumotion version %s" % configure.version)
import twisted.copyright
log.debug("manager", "Running against Twisted version %s" % twisted.copyright.version)
from flumotion.project import project
for p in project.list():
log.debug("manager", "Registered project %s version %s" % (p, project.get(p, "version")))
vishnu = manager.Vishnu(options.name, configDir=managerConfigDir)
for managerConfigFile in args[1:]:
vishnu.loadManagerConfigurationXML(managerConfigFile)
paths = [os.path.abspath(filename) for filename in args[1:]]
# logging
flumotion.common.setup.setup()
from flumotion.common import log
def useGtk2Reactor():
var = 'FLU_TEST_GTK2_REACTOR'
if var not in os.environ:
return False
else:
return True
if useGtk2Reactor():
log.info('check', 'using gtk2 reactor')
from twisted.internet import gtk2reactor
gtk2reactor.install()
else:
log.info('check', 'using default reactor')
# have to choose the reactor before calling this method
log.logTwisted()
# FIXME: boot.py does this, but enabling this borks
# test_common_package.py. I have no idea what that code does, either.
#
# # installing the reactor could override our packager's import hooks ...
# from twisted.internet import reactor
# # ... so we install them again here to be safe
# from flumotion.common import package
def main(args):
parser = _createParser()
log.debug('worker', 'Parsing arguments (%r)' % ', '.join(args))
options, args = parser.parse_args(args)
# Force options down configure's throat
for d in ['logdir', 'rundir']:
o = getattr(options, d, None)
if o:
log.debug('worker', 'Setting configure.%s to %s' % (d, o))
setattr(configure, d, o)
# translate feederports string to range
if options.feederports:
if not '-' in options.feederports:
raise errors.OptionError("feederports '%s' does not contain '-'" %
options.feederports)
(lower, upper) = options.feederports.split('-')
options.feederports = range(int(lower), int(upper) + 1)
# check if a config file was specified; if so, parse config and copy over
if len(args) > 1:
workerFile = args[1]
_readConfig(workerFile, options)
# set default values for all unset options
if not options.host:
options.host = 'localhost'
if not options.transport:
options.transport = 'ssl'
if not options.port:
if options.transport == "tcp":
options.port = configure.defaultTCPManagerPort
elif options.transport == "ssl":
options.port = configure.defaultSSLManagerPort
# set a default name if none is given
if not options.name:
if options.host == 'localhost':
options.name = 'localhost'
log.debug('worker', 'Setting worker name localhost')
else:
import socket
options.name = socket.gethostname()
log.debug('worker', 'Setting worker name %s (from hostname)' %
options.name)
if options.feederports is None and not options.randomFeederports:
options.feederports = configure.defaultGstPortRange
log.debug('worker', 'Using default feederports %r' %
options.feederports)
# check for wrong options/arguments
if not options.transport in ['ssl', 'tcp']:
sys.stderr.write('ERROR: wrong transport %s, must be ssl or tcp\n' %
options.transport)
return 1
# reset FLU_DEBUG which could be different after parsing XML file
if options.debug:
log.setFluDebug(options.debug)
if options.daemonizeTo and not options.daemonize:
sys.stderr.write(
'ERROR: --daemonize-to can only be used with -D/--daemonize.\n')
return 1
if options.serviceName and not options.daemonize:
sys.stderr.write(
'ERROR: --service-name can only be used with -D/--daemonize.\n')
return 1
brain = worker.WorkerBrain(options)
# Now bind and listen to our unix and tcp sockets
if not brain.listen():
sys.stderr.write('ERROR: Failed to listen on worker ports.\n')
return 1
name = options.name
if options.daemonize:
if options.serviceName:
name = options.serviceName
if not options.daemonizeTo:
options.daemonizeTo = "/"
startup("worker", name, options.daemonize, options.daemonizeTo)
log.debug('worker', 'Running Flumotion version %s' %
configure.version)
import twisted.copyright
log.debug('worker', 'Running against Twisted version %s' %
twisted.copyright.version)
# register all package paths (FIXME: this should go away when
# components come from manager)
from flumotion.common import setup
setup.setupPackagePath()
# FIXME: why address='localhost' ?
authenticator = pb.Authenticator(username=options.username,
password=options.password,
address='localhost',
avatarId=options.name)
info = connection.PBConnectionInfo(options.host, options.port,
options.transport == "ssl",
authenticator)
brain.login(info)
log.info('worker',
'Connecting to manager %s using %s' % (info,
options.transport.upper()))
# go into the reactor main loop
reactor.run()
return 0
if 'FLU_PROFILE' in os.environ:
try:
import statprof
statprof.start()
print 'Profiling started.'
def stop_profiling():
statprof.stop()
statprof.display()
reactor.addSystemEventTrigger('before', 'shutdown',
stop_profiling)
except ImportError, e:
print ('Profiling requested, but statprof is not available (%s)'
% e)
reactor.addSystemEventTrigger('before', 'shutdown',
job_factory.medium.shutdownHandler)
# log our standardized started marker
log.info('job', "Started job '%s'" % avatarId)
reactor.run()
# log our standardized stopping marker
log.info('job', "Stopping job '%s'" % avatarId)
# log our standardized stopped marker
log.info('job', "Stopped job '%s'" % avatarId)
return 0
log.debug('manager', 'Setting name to unnamed')
else:
options.name = name
log.debug('manager', 'Setting name to %s based on path' % name)
# check for wrong options/arguments
if not options.transport in ['ssl', 'tcp']:
sys.stderr.write('ERROR: wrong transport %s, must be ssl or tcp\n' %
options.transport)
return 1
# register package path
setup.setupPackagePath()
# log our standardized starting marker
log.info('manager', "Starting manager '%s'" % options.name)
log.debug('manager', 'Running Flumotion version %s' %
configure.version)
import twisted.copyright
log.debug('manager', 'Running against Twisted version %s' %
twisted.copyright.version)
from flumotion.project import project
for p in project.list():
log.debug('manager', 'Registered project %s version %s' % (
p, project.get(p, 'version')))
vishnu = manager.Vishnu(options.name, configDir=managerConfigDir)
for managerConfigFile in args[1:]:
vishnu.loadManagerConfigurationXML(managerConfigFile)
def main(args):
parser = _createParser()
log.debug('worker', 'Parsing arguments (%r)' % ', '.join(args))
options, args = parser.parse_args(args)
# Force options down configure's throat
for d in ['logdir', 'rundir']:
o = getattr(options, d, None)
if o:
log.debug('worker', 'Setting configure.%s to %s' % (d, o))
setattr(configure, d, o)
# translate feederports string to range
if options.feederports:
if not '-' in options.feederports:
raise errors.OptionError("feederports '%s' does not contain '-'" %
options.feederports)
(lower, upper) = options.feederports.split('-')
options.feederports = range(int(lower), int(upper) + 1)
# check if a config file was specified; if so, parse config and copy over
if len(args) > 1:
workerFile = args[1]
_readConfig(workerFile, options)
# set default values for all unset options
if not options.host:
options.host = 'localhost'
if not options.transport:
options.transport = 'ssl'
if not options.port:
if options.transport == "tcp":
options.port = configure.defaultTCPManagerPort
elif options.transport == "ssl":
options.port = configure.defaultSSLManagerPort
# set a default name if none is given
if not options.name:
if options.host == 'localhost':
options.name = 'localhost'
log.debug('worker', 'Setting worker name localhost')
else:
import socket
options.name = socket.gethostname()
log.debug('worker',
'Setting worker name %s (from hostname)' % options.name)
if options.feederports is None and not options.randomFeederports:
options.feederports = configure.defaultGstPortRange
log.debug('worker',
'Using default feederports %r' % options.feederports)
# check for wrong options/arguments
if not options.transport in ['ssl', 'tcp']:
sys.stderr.write('ERROR: wrong transport %s, must be ssl or tcp\n' %
options.transport)
return 1
# reset FLU_DEBUG which could be different after parsing XML file
if options.debug:
log.setFluDebug(options.debug)
if options.daemonizeTo and not options.daemonize:
sys.stderr.write(
'ERROR: --daemonize-to can only be used with -D/--daemonize.\n')
return 1
if options.serviceName and not options.daemonize:
sys.stderr.write(
'ERROR: --service-name can only be used with -D/--daemonize.\n')
return 1
brain = worker.WorkerBrain(options)
# Now bind and listen to our unix and tcp sockets
if not brain.listen():
sys.stderr.write('ERROR: Failed to listen on worker ports.\n')
return 1
name = options.name
if options.daemonize:
if options.serviceName:
name = options.serviceName
if not options.daemonizeTo:
options.daemonizeTo = "/"
startup("worker", name, options.daemonize, options.daemonizeTo)
log.debug('worker', 'Running Flumotion version %s' % configure.version)
import twisted.copyright
log.debug('worker',
'Running against Twisted version %s' % twisted.copyright.version)
# register all package paths (FIXME: this should go away when
# components come from manager)
from flumotion.common import setup
setup.setupPackagePath()
# FIXME: why address='localhost' ?
authenticator = pb.Authenticator(username=options.username,
password=options.password,
address='localhost',
avatarId=options.name)
info = connection.PBConnectionInfo(options.host, options.port,
options.transport == "ssl",
authenticator)
brain.login(info)
log.info(
'worker', 'Connecting to manager %s using %s' %
(info, options.transport.upper()))
# go into the reactor main loop
reactor.run()
return 0
def shutdownStarted():
log.info(processType, "Stopping %s '%s'", processType, processName)
def shutdownEnded():
log.info(processType, "Stopped %s '%s'", processType, processName)
flumotion.common.setup.setup()
from flumotion.common import log
def useGtk2Reactor():
var = "FLU_TEST_GTK2_REACTOR"
if var not in os.environ:
return False
else:
return True
if useGtk2Reactor():
log.info("check", "using gtk2 reactor")
from twisted.internet import gtk2reactor
gtk2reactor.install()
else:
log.info("check", "using default reactor")
# have to choose the reactor before calling this method
log.logTwisted()
# FIXME: boot.py does this, but enabling this borks
# test_common_package.py. I have no idea what that code does, either.
#
# # installing the reactor could override our packager's import hooks ...
# from twisted.internet import reactor
# # ... so we install them again here to be safe
def registerGIOJelly():
"""Register the jelly used by the GIO VFS backend.
"""
setUnjellyableForClass(GIOFile, GIOFile)
setUnjellyableForClass(GIODirectory, GIODirectory)
log.info('jelly', 'GIO registered')
# logging
flumotion.common.setup.setup()
from flumotion.common import log
def useGtk3Reactor():
var = 'FLU_TEST_GTK3_REACTOR'
if var not in os.environ:
return False
else:
return True
if useGtk3Reactor():
log.info('check', 'using gtk3 reactor')
from twisted.internet import gtk3reactor
gtk3reactor.install()
else:
log.info('check', 'using default reactor')
# have to choose the reactor before calling this method
log.logTwisted()
# FIXME: boot.py does this, but enabling this borks
# test_common_package.py. I have no idea what that code does, either.
#
# # installing the reactor could override our packager's import hooks ...
# from twisted.internet import reactor
# # ... so we install them again here to be safe
# from flumotion.common import package
def registerGnomeVFSJelly():
"""Register the jelly used by the GnomeVFS VFS backend.
"""
setUnjellyableForClass(GnomeVFSFile, GnomeVFSFile)
setUnjellyableForClass(GnomeVFSDirectory, GnomeVFSDirectory)
log.info('jelly', 'GnomeVFS registered')