def main():
parser = optparse.OptionParser()
parser.add_option('--pulse-cfg',
dest='pulse_cfg',
default='',
help='Pulse config')
parser.add_option('--pulse-cfg-section',
dest='pulse_cfg_section',
default='pulse',
help='Pulse config section')
options, args = parser.parse_args()
pulse = NormalizedBuildConsumer(applabel='translator_test_consumer_%s' %
uuid.uuid4(),
connect=False)
pulse.configure(topic=['build.#', 'unittest.#', 'talos.#'],
callback=on_pulse_message,
durable=False)
if options.pulse_cfg:
if not os.path.exists(options.pulse_cfg):
print 'Config file does not exist!'
return
cfg = ConfigParser.ConfigParser()
cfg.read(options.pulse_cfg)
pulse.config = PulseConfiguration.read_from_config(
cfg, options.pulse_cfg_section)
pulse.listen()
def __init__(self,
configfile,
pulse_authfile,
debug,
log_folder,
logger,
message=None,
display_only=False):
self.config = JSONFile(configfile).read()
self.debug = debug
self.log_folder = log_folder
self.logger = logger
self.display_only = display_only
self.test_message = message
self.jenkins = jenkins.Jenkins(self.config['jenkins']['url'],
self.config['jenkins']['username'],
self.config['jenkins']['password'])
# When a local pulse message is used, return immediately
if self.test_message:
data = JSONFile(self.test_message).read()
self.on_build(data)
return
# Load Pulse Guardian authentication from config file
if not os.path.exists(pulse_authfile):
print 'Config file for Mozilla Pulse does not exist!'
return
pulse_cfgfile = ConfigParser.ConfigParser()
pulse_cfgfile.read(pulse_authfile)
pulse_cfg = PulseConfiguration.read_from_config(pulse_cfgfile)
label = '%s/%s' % (socket.getfqdn(), self.config['pulse']['applabel'])
self.monitor = start_pulse_monitor(
buildCallback=self.on_build,
testCallback=None,
pulseCallback=self.on_debug if self.debug else None,
label=label,
trees=self.config['pulse']['branches'],
platforms=self.config['pulse']['platforms'],
products=self.config['pulse']['products'],
buildtypes=None,
tests=None,
buildtags=self.config['pulse']['tags'],
logger=self.logger,
pulse_cfg=pulse_cfg)
try:
while self.monitor.is_alive():
self.monitor.join(1.0)
except (KeyboardInterrupt, SystemExit):
self.logger.info('Shutting down Pulse listener')
def __init__(self, cfg, cb):
threading.Thread.__init__(self)
self.cb = cb
self.consumer = GenericConsumer(
PulseConfiguration(**cfg),
['exchange/bugzilla/simple', 'exchange/bugzilla/simple/dev'],
**cfg)
def main():
parser = optparse.OptionParser()
parser.add_option('--pidfile', dest='pidfile',
default='translator.pid',
help='path to file for logging pid')
parser.add_option('--logfile', dest='logfile',
default='stdout.log',
help='path to file for stdout logging')
parser.add_option('--logdir', dest='logdir',
default='logs',
help='directory to store other log files')
parser.add_option('--daemon', dest='daemon', action='store_true',
help='run as daemon (posix only)')
parser.add_option('--durable',
dest='durable',
action='store_true',
default=False,
help='register a durable queue')
parser.add_option('--display-only',
dest='display_only',
action='store_true',
default=False,
help='only display build properties and don\'t add '
'jobs to the queue')
parser.add_option('--pulse-cfg',
dest='pulse_cfg',
default='',
help='optional config file containing optional sections '
'[consumer] and [publisher] for nondefault Pulse '
'configs')
options, args = parser.parse_args()
pulse_cfgs = {'consumer': None, 'publisher': None}
if options.pulse_cfg:
if not os.path.exists(options.pulse_cfg):
print 'Config file does not exist!'
return
pulse_cfgfile = ConfigParser.ConfigParser()
pulse_cfgfile.read(options.pulse_cfg)
for section in pulse_cfgs.keys():
pulse_cfgs[section] = PulseConfiguration.read_from_config(
pulse_cfgfile, section)
if options.daemon:
if os.access(options.logfile, os.F_OK):
os.remove(options.logfile)
createDaemon(options.pidfile, options.logfile)
f = open(options.pidfile, 'w')
f.write("%d\n" % os.getpid())
f.close()
service = PulseBuildbotTranslator(durable=options.durable,
logdir=options.logdir,
display_only=options.display_only,
consumer_cfg=pulse_cfgs['consumer'],
publisher_cfg=pulse_cfgs['publisher'])
service.start()
def test_config(self):
cfg = ConfigParser.ConfigParser()
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.verify_defaults(pulse_cfg)
cfg.add_section('pulse')
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.verify_defaults(pulse_cfg)
cfg.set('pulse', 'user', 'foo')
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.assertEqual(pulse_cfg.user, 'foo')
self.assertEqual(pulse_cfg.port, DEFAULT_SSL_PORT)
self.verify_defaults(pulse_cfg, ['user'])
cfg.set('pulse', 'ssl', 'off')
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.assertEqual(pulse_cfg.ssl, False)
self.assertEqual(pulse_cfg.port, DEFAULT_PORT)
self.verify_defaults(pulse_cfg, ['user', 'ssl', 'port'])
cfg.set('pulse', 'port', '5555')
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.assertEqual(pulse_cfg.port, 5555)
self.verify_defaults(pulse_cfg, ['user', 'ssl', 'port'])
def test_config(self):
cfg = configparser.ConfigParser()
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.verify_defaults(pulse_cfg)
cfg.add_section('pulse')
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.verify_defaults(pulse_cfg)
cfg.set('pulse', 'user', 'foo')
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.assertEqual(pulse_cfg.user, 'foo')
self.assertEqual(pulse_cfg.port, DEFAULT_SSL_PORT)
self.verify_defaults(pulse_cfg, ['user'])
cfg.set('pulse', 'ssl', 'off')
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.assertEqual(pulse_cfg.ssl, False)
self.assertEqual(pulse_cfg.port, DEFAULT_PORT)
self.verify_defaults(pulse_cfg, ['user', 'ssl', 'port'])
cfg.set('pulse', 'port', '5555')
pulse_cfg = PulseConfiguration.read_from_config(cfg)
self.assertEqual(pulse_cfg.port, 5555)
self.verify_defaults(pulse_cfg, ['user', 'ssl', 'port'])
def __init__(self, configfile, pulse_authfile, debug, log_folder, logger,
message=None, display_only=False):
self.config = JSONFile(configfile).read()
self.debug = debug
self.log_folder = log_folder
self.logger = logger
self.display_only = display_only
self.test_message = message
self.jenkins = jenkins.Jenkins(self.config['jenkins']['url'],
self.config['jenkins']['username'],
self.config['jenkins']['password'])
# When a local pulse message is used, return immediately
if self.test_message:
data = JSONFile(self.test_message).read()
self.on_build(data)
return
# Load Pulse Guardian authentication from config file
if not os.path.exists(pulse_authfile):
print 'Config file for Mozilla Pulse does not exist!'
return
pulse_cfgfile = ConfigParser.ConfigParser()
pulse_cfgfile.read(pulse_authfile)
pulse_cfg = PulseConfiguration.read_from_config(pulse_cfgfile)
label = '%s/%s' % (socket.getfqdn(), self.config['pulse']['applabel'])
self.monitor = start_pulse_monitor(buildCallback=self.on_build,
testCallback=None,
pulseCallback=self.on_debug if self.debug else None,
label=label,
trees=self.config['pulse']['branches'],
platforms=self.config['pulse']['platforms'],
products=self.config['pulse']['products'],
buildtypes=None,
tests=None,
buildtags=self.config['pulse']['tags'],
logger=self.logger,
pulse_cfg=pulse_cfg)
try:
while self.monitor.is_alive():
self.monitor.join(1.0)
except (KeyboardInterrupt, SystemExit):
self.logger.info('Shutting down Pulse listener')
def main():
parser = optparse.OptionParser()
parser.add_option('--pulse-cfg', dest='pulse_cfg', default='',
help='Pulse config')
parser.add_option('--pulse-cfg-section', dest='pulse_cfg_section',
default='pulse', help='Pulse config section')
options, args = parser.parse_args()
pulse = NormalizedBuildConsumer(applabel='translator_test_consumer_%s'
% uuid.uuid4(), connect=False)
pulse.configure(topic=['build.#', 'unittest.#', 'talos.#'],
callback=on_pulse_message,
durable=False)
if options.pulse_cfg:
if not os.path.exists(options.pulse_cfg):
print 'Config file does not exist!'
return
cfg = ConfigParser.ConfigParser()
cfg.read(options.pulse_cfg)
pulse.config = PulseConfiguration.read_from_config(
cfg, options.pulse_cfg_section)
pulse.listen()
def __init__(self, exchange, **kwargs):
super(PulseConsumer, self).__init__(
PulseConfiguration(**kwargs), exchange, **kwargs)
def __init__(self, **kwargs):
super(B2GPulseConsumer,
self).__init__(PulseConfiguration(**kwargs),
'org.mozilla.exchange.b2g', **kwargs)
def main():
from optparse import OptionParser
from mozillapulse.publishers import GenericPublisher
from mozillapulse.config import PulseConfiguration
import logging.handlers
parser = OptionParser()
parser.set_defaults(
verbosity=0,
logfile=None,
max_retries=5,
retry_time=60,
)
parser.add_option("--passwords", dest="passwords")
parser.add_option("-q", "--queuedir", dest="queuedir")
parser.add_option("-v",
"--verbose",
dest="verbosity",
action="count",
help="increase verbosity")
parser.add_option("-l",
"--logfile",
dest="logfile",
help="where to send logs")
parser.add_option("-r",
"--max_retries",
dest="max_retries",
type="int",
help="number of times to retry")
parser.add_option("-t",
"--retry_time",
dest="retry_time",
type="int",
help="seconds to wait between retries")
options, args = parser.parse_args()
# Set up logging
if options.verbosity == 0:
log_level = logging.WARNING
elif options.verbosity == 1:
log_level = logging.INFO
else:
log_level = logging.DEBUG
if not options.logfile:
logging.basicConfig(level=log_level,
format="%(asctime)s - %(message)s")
else:
logger = logging.getLogger()
logger.setLevel(log_level)
handler = logging.handlers.RotatingFileHandler(options.logfile,
maxBytes=1024**2,
backupCount=5)
formatter = logging.Formatter("%(asctime)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
if not options.passwords:
parser.error("--passwords is required")
if not options.queuedir:
parser.error("-q/--queuedir is required")
passwords = {}
execfile(options.passwords, passwords, passwords)
publisher = GenericPublisher(PulseConfiguration(
user=passwords['PULSE_USERNAME'],
password=passwords['PULSE_PASSWORD'],
),
exchange=passwords['PULSE_EXCHANGE'])
pusher = PulsePusher(options.queuedir,
publisher,
max_retries=options.max_retries,
retry_time=options.retry_time)
pusher.loop()
def __init__(self, **kwargs):
super(PulseMetaConsumer, self).__init__(PulseConfiguration(**kwargs),
'exchange/pulse/', **kwargs)
def main():
parser = optparse.OptionParser()
parser.add_option('--pidfile',
dest='pidfile',
default='translator.pid',
help='path to file for logging pid')
parser.add_option('--logfile',
dest='logfile',
default='stdout.log',
help='path to file for stdout logging')
parser.add_option('--logdir',
dest='logdir',
default='logs',
help='directory to store other log files')
parser.add_option('--daemon',
dest='daemon',
action='store_true',
help='run as daemon (posix only)')
parser.add_option('--durable',
dest='durable',
action='store_true',
default=False,
help='register a durable queue')
parser.add_option('--display-only',
dest='display_only',
action='store_true',
default=False,
help='only display build properties and don\'t add '
'jobs to the queue')
parser.add_option('--pulse-cfg',
dest='pulse_cfg',
default='',
help='optional config file containing optional sections '
'[consumer] and [publisher] for nondefault Pulse '
'configs')
parser.add_option('--push-message',
dest='message',
help='path to file of a Pulse message to process')
parser.add_option('--label',
dest='label',
help='label to use for pulse queue')
options, args = parser.parse_args()
pulse_cfgs = {'consumer': None, 'publisher': None}
if options.pulse_cfg:
if not os.path.exists(options.pulse_cfg):
print 'Config file does not exist!'
return
pulse_cfgfile = ConfigParser.ConfigParser()
pulse_cfgfile.read(options.pulse_cfg)
for section in pulse_cfgs.keys():
pulse_cfgs[section] = PulseConfiguration.read_from_config(
pulse_cfgfile, section)
if os.environ.get('pulseuser'):
setattr(pulse_cfgs['consumer'], 'user', os.environ['pulseuser'])
setattr(pulse_cfgs['publisher'], 'user', os.environ['pulseuser'])
if os.environ.get('pulsepassword'):
setattr(pulse_cfgs['consumer'], 'password',
os.environ['pulsepassword'])
setattr(pulse_cfgs['publisher'], 'password',
os.environ['pulsepassword'])
if options.daemon:
if os.access(options.logfile, os.F_OK):
os.remove(options.logfile)
createDaemon(options.pidfile, options.logfile)
f = open(options.pidfile, 'w')
f.write("%d\n" % os.getpid())
f.close()
service = PulseBuildbotTranslator(durable=options.durable,
logdir=options.logdir,
message=options.message,
label=options.label,
display_only=options.display_only,
consumer_cfg=pulse_cfgs['consumer'],
publisher_cfg=pulse_cfgs['publisher'])
service.start()
def __init__(self, **kwargs):
exchange = 'exchange/bugzilla/simple'
if kwargs.get('dev'):
exchange += '/dev'
super(SimpleBugzillaConsumer,
self).__init__(PulseConfiguration(**kwargs), exchange, **kwargs)
def __init__(self, **kwargs):
super(MozReviewConsumer,
self).__init__(PulseConfiguration(**kwargs),
'exchange/mozreview/', **kwargs)
def __init__(self, **kwargs):
super(QAConsumer, self).__init__(PulseConfiguration(**kwargs),
'exchange/qa/', **kwargs)
def __init__(self, **kwargs):
super(NormalizedBuildConsumer,
self).__init__(PulseConfiguration(**kwargs),
'exchange/build/normalized', **kwargs)
