def main():
parser = argparse.ArgumentParser(description='Marteau Server')
parser.add_argument('config', help='Config file', nargs='?')
parser.add_argument('--version', action='store_true',
default=False,
help='Displays Marteau version and exits.')
parser.add_argument('--log-level', dest='loglevel', default='info',
choices=LOG_LEVELS.keys() + [key.upper() for key in
LOG_LEVELS.keys()],
help="log level")
parser.add_argument('--log-output', dest='logoutput', default='-',
help="log output")
parser.add_argument('--host', help='Host', default='0.0.0.0')
parser.add_argument('--port', help='Port', type=int, default=8080)
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
# configure the logger
configure_logger(logger, args.loglevel, args.logoutput)
# loading the config file
config = Config(args.config)
# loading the app & the queue
global_config = {}
if config.has_section('marteau'):
settings = config.get_map('marteau')
else:
settings = {}
# check is redis is running
if not redis_available():
raise IOError('Marteau needs Redis to run.')
# loading the fixtures plugins
for fixture in settings.get('fixtures', []):
import_string(fixture)
logger.info('Loaded plugins: %s' % ', '.join(get_fixtures()))
app = webapp(global_config, **settings)
try:
httpd = SocketIOServer((args.host, args.port), app,
resource="socket.io", policy_server=False)
logger.info('Hammer ready, at http://%s:%s. Where are the nails ?' %
(args.host, args.port))
httpd.serve_forever()
except KeyboardInterrupt:
sys.exit(0)
finally:
logger.info('Bye!')
def main():
parser = argparse.ArgumentParser(description='Funkload Server')
parser.add_argument('--config', help='Config file, if any')
parser.add_argument('--version', action='store_true',
default=False,
help='Displays Marteau version and exits.')
parser.add_argument('--log-level', dest='loglevel', default='info',
choices=LOG_LEVELS.keys() + [key.upper() for key in
LOG_LEVELS.keys()],
help="log level")
parser.add_argument('--log-output', dest='logoutput', default='-',
help="log output")
parser.add_argument('--host', help='Host', default='0.0.0.0')
parser.add_argument('--port', help='Port', type=int, default=8080)
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
# configure the logger
configure_logger(logger, args.loglevel, args.logoutput)
# loading the config file
config = Config()
if args.config is not None:
logger.info('Loading %r' % args.config)
config.read([args.config])
# loading the app & the queue
global_config = {}
if config.has_section('marteau'):
settings = config.get_map('marteau')
else:
settings = {}
# loading the fixtures plugins
for fixture in settings.get('fixtures', []):
import_string(fixture)
logger.info('Loaded plugins: %s' % ', '.join(get_fixtures()))
app = webapp(global_config, **settings)
try:
httpd = make_server(args.host, args.port, app)
logger.info('Hammer ready, at http://%s:%s. Where are the nails ?' %
(args.host, args.port))
httpd.serve_forever()
except KeyboardInterrupt:
sys.exit(0)
finally:
logger.info('Bye!')
def run_loadtest(repo,
cycles=None,
nodes_count=None,
duration=None,
email=None,
options=None,
distributed=True,
fl_result_path=None,
queue=None,
fixture_plugin=None,
fixture_options=None,
workdir=DEFAULT_WORKDIR,
reportsdir=DEFAULT_REPORTSDIR,
test=None,
script=None):
if options is None:
options = {}
rtfeedback = options.get('feedback', None) is not None
# loading the fixtures plugins
for fixture in options.get('fixtures', []):
import_string(fixture)
job_id = os.environ.get('MARTEAU_JOBID', '')
if options is None:
options = {}
if os.path.exists(repo):
# just a local dir, lets work there
os.chdir(repo)
_logrun('Moved to %r' % repo)
target = os.path.realpath(repo)
else:
# checking out the repo
os.chdir(workdir)
name = repo.split('/')[-1].split('.')[0]
target = os.path.join(workdir, name)
if os.path.exists(target):
os.chdir(target)
_logrun('Moved to %r' % target)
run_func(queue, job_id, 'git pull')
else:
_logrun('Moved to %r' % workdir)
run_func(queue,
job_id,
'git clone %s' % repo,
stop_on_failure=False)
os.chdir(target)
# now looking for the marteau config file in there
config = read_yaml_config(os.getcwd())
wdir = config.get('wdir')
if wdir is not None:
target = os.path.join(target, wdir)
os.chdir(target)
deps = config.get('deps', [])
if rtfeedback and 'pyzmq' not in deps:
deps.append('pyzmq')
if distributed:
# is this a distributed test ?
if nodes_count in (None, ''): # XXX fix later
nodes_count = config.get('nodes', 1)
else:
nodes_count = int(nodes_count)
nodes = get_nodes(nodes_count, queue, options)
nodes_names = ','.join([node.name for node in nodes])
os.environ['MARTEAU_NODES'] = nodes_names
workers = '--distribute-workers=%s' % nodes_names
cmd = '%s --distribute %s' % (run_bench, workers)
if deps != []:
cmd += ' --distributed-packages="%s"' % ' '.join(deps)
target = tempfile.mkdtemp()
cmd += ' --distributed-log-path=%s' % target
if 'ssh_key' in options:
cmd += ' --distributed-key-filename=%s' % options['ssh_key']
# asking the node to send us realtime feedback.
if rtfeedback:
cmd += ' --feedback'
cmd += ' --feedback-endpoint %s' % options['feedback_endpoint']
cmd += ' --feedback-pubsub-endpoint %s' % \
options['feedback_publisher']
else:
cmd = run_bench
try:
# creating a virtualenv there
run_func(queue, job_id, 'virtualenv --no-site-packages .')
run_func(queue, job_id, run_pip + ' install funkload')
# install dependencies if any
for dep in deps:
run_func(queue, job_id, run_pip + ' install %s' % dep)
if fl_result_path is not None:
# in funkload this is a relative path
target = os.path.join(target, fl_result_path)
xml_files = os.path.join(target, '*.xml')
if cycles is None:
cycles = config.get('cycles')
if cycles is not None:
cmd += ' --cycles=%s' % cycles
if duration is None:
duration = config.get('duration')
if test is None:
test = config.get('test')
if script is None:
script = config.get('script')
if duration is not None:
cmd += ' --duration=%s' % duration
report_dir = os.path.join(reportsdir,
os.environ.get('MARTEAU_JOBID', 'report'))
if fixture_plugin:
_logrun('Running the %r fixture' % fixture_plugin)
fixture_klass = get_fixture(fixture_plugin)
if fixture_options is None:
fixture_options = {}
try:
fixture = fixture_klass(**fixture_options)
except:
_logrun('Could not instanciate a fixture plugin instance')
raise
try:
fixture.setup()
except:
_logrun('The fixture set up failed')
raise
# starting the feedback subscriberin its own thread
if rtfeedback:
from funkload.rtfeedback import FeedbackSubscriber
sub = FeedbackSubscriber(
pubsub_endpoint=options['feedback_publisher'],
handler=_rt_handler,
pid=os.getpid(),
queue=queue)
sub.start()
try:
_logrun('Running the loadtest')
run_func(queue, job_id, '%s %s %s' % (cmd, script, test))
finally:
_logrun('Running the fixture tear_down method')
if fixture_plugin:
try:
fixture.tear_down()
except:
_logrun('The fixture tear down failed')
raise
if rtfeedback:
sub.terminate()
_logrun('Building the report')
report = run_report + ' --skip-definitions --css %s --html -r %s %s'
run_func(queue, job_id, report % (CSS_FILE, report_dir, xml_files))
# do we send an email with the result ?
if email is None:
email = config.get('email')
if email is not None:
_logrun('Sending an e-mail to %r' % email)
try:
res, msg = send_report(email, job_id, **options)
except Exception, e:
res = False
msg = str(e)
if not res:
_logrun(msg)
else:
_logrun('Mail sent.')
return report_dir
def run_loadtest(repo, cycles=None, nodes_count=None, duration=None,
email=None, options=None, distributed=True,
queue=None, fixture_plugin=None, fixture_options=None,
workdir=DEFAULT_WORKDIR, reportsdir=DEFAULT_REPORTSDIR,
test=None, script=None):
if options is None:
options = {}
# loading the fixtures plugins
for fixture in options.get('fixtures', []):
import_string(fixture)
job_id = os.environ.get('MARTEAU_JOBID', '')
if options is None:
options = {}
if os.path.exists(repo):
# just a local dir, lets work there
os.chdir(repo)
_logrun('Moved to %r' % repo)
target = os.path.realpath(repo)
else:
# checking out the repo
os.chdir(workdir)
name = repo.split('/')[-1].split('.')[0]
target = os.path.join(workdir, name)
if os.path.exists(target):
os.chdir(target)
_logrun('Moved to %r' % target)
run_func(queue, job_id, 'git pull')
else:
_logrun('Moved to %r' % workdir)
run_func(queue, job_id,
'git clone %s' % repo, stop_on_failure=False)
os.chdir(target)
# now looking for the marteau config file in there
config = read_yaml_config(os.getcwd())
wdir = config.get('wdir')
if wdir is not None:
target = os.path.join(target, wdir)
os.chdir(target)
deps = config.get('deps', [])
if distributed:
# is this a distributed test ?
if nodes_count in (None, ''): # XXX fix later
nodes_count = config.get('nodes', 1)
else:
nodes_count = int(nodes_count)
# we want to pick up the number of nodes asked
nodes = queue.get_nodes(check_available=True)
if len(nodes) < nodes_count:
# we want to pile this one back and sleep a bit here
_logrun('Sleeping for 30 s.')
time.sleep(30)
raise ValueError("Sorry could not find enough free nodes")
# then pick random ones
random.shuffle(nodes)
nodes = nodes[:nodes_count]
# save the nodes status
for node in nodes:
node.status = 'working'
queue.save_node(node)
workers = ','.join([node.name for node in nodes])
os.environ['MARTEAU_NODES'] = workers
workers = '--distribute-workers=%s' % workers
cmd = '%s --distribute %s' % (run_bench, workers)
if deps != []:
cmd += ' --distributed-packages="%s"' % ' '.join(deps)
target = tempfile.mkdtemp()
cmd += ' --distributed-log-path=%s' % target
else:
cmd = run_bench
# creating a virtualenv there
run_func(queue, job_id, 'virtualenv --no-site-packages .')
run_func(queue, job_id, run_pip + ' install funkload')
# install dependencies if any
for dep in deps:
run_func(queue, job_id, run_pip + ' install %s' % dep)
xml_files = os.path.join(target, '*.xml')
if cycles is None:
cycles = config.get('cycles')
if cycles is not None:
cmd += ' --cycles=%s' % cycles
if duration is None:
duration = config.get('duration')
if test is None:
test = config.get('test')
if script is None:
script = config.get('script')
if duration is not None:
cmd += ' --duration=%s' % duration
report_dir = os.path.join(reportsdir,
os.environ.get('MARTEAU_JOBID', 'report'))
if fixture_plugin:
_logrun('Running the %r fixture' % fixture_plugin)
fixture_klass = get_fixture(fixture_plugin)
if fixture_options is None:
fixture_options = {}
try:
fixture = fixture_klass(**fixture_options)
except:
_logrun('Could not instanciate a fixture plugin instance')
raise
try:
fixture.setup()
except:
_logrun('The fixture set up failed')
raise
try:
_logrun('Running the loadtest')
run_func(queue, job_id, '%s %s %s' % (cmd, script, test))
finally:
_logrun('Running the fixture tear_down method')
if fixture_plugin:
try:
fixture.tear_down()
except:
_logrun('The fixture tear down failed')
raise
_logrun('Building the report')
report = run_report + ' --skip-definitions --css %s --html -r %s %s'
run_func(queue, job_id, report % (CSS_FILE, report_dir, xml_files))
# do we send an email with the result ?
if email is None:
email = config.get('email')
if email is not None:
_logrun('Sending an e-mail to %r' % email)
try:
res, msg = send_report(email, job_id, **options)
except Exception, e:
res = False
msg = str(e)
if not res:
_logrun(msg)
else:
_logrun('Mail sent.')
def run_loadtest(repo, cycles=None, nodes_count=None, duration=None,
email=None, options=None, distributed=True,
fl_result_path=None, queue=None, fixture_plugin=None,
fixture_options=None, workdir=DEFAULT_WORKDIR,
reportsdir=DEFAULT_REPORTSDIR, test=None, script=None):
if options is None:
options = {}
rtfeedback = options.get('feedback', None) is not None
# loading the fixtures plugins
for fixture in options.get('fixtures', []):
import_string(fixture)
job_id = os.environ.get('MARTEAU_JOBID', '')
if options is None:
options = {}
if os.path.exists(repo):
# just a local dir, lets work there
os.chdir(repo)
_logrun('Moved to %r' % repo)
target = os.path.realpath(repo)
else:
# checking out the repo
os.chdir(workdir)
name = repo.split('/')[-1].split('.')[0]
target = os.path.join(workdir, name)
if os.path.exists(target):
os.chdir(target)
_logrun('Moved to %r' % target)
run_func(queue, job_id, 'git pull')
else:
_logrun('Moved to %r' % workdir)
run_func(queue, job_id,
'git clone %s' % repo, stop_on_failure=False)
os.chdir(target)
# now looking for the marteau config file in there
config = read_yaml_config(os.getcwd())
wdir = config.get('wdir')
if wdir is not None:
target = os.path.join(target, wdir)
os.chdir(target)
deps = config.get('deps', [])
if rtfeedback and 'pyzmq' not in deps:
deps.append('pyzmq')
if distributed:
# is this a distributed test ?
if nodes_count in (None, ''): # XXX fix later
nodes_count = config.get('nodes', 1)
else:
nodes_count = int(nodes_count)
nodes = get_nodes(nodes_count, queue, options)
nodes_names = ','.join([node.name for node in nodes])
os.environ['MARTEAU_NODES'] = nodes_names
workers = '--distribute-workers=%s' % nodes_names
cmd = '%s --distribute %s' % (run_bench, workers)
if deps != []:
cmd += ' --distributed-packages="%s"' % ' '.join(deps)
target = tempfile.mkdtemp()
cmd += ' --distributed-log-path=%s' % target
if 'ssh_key' in options:
cmd += ' --distributed-key-filename=%s' % options['ssh_key']
# asking the node to send us realtime feedback.
if rtfeedback:
cmd += ' --feedback'
cmd += ' --feedback-endpoint %s' % options['feedback_endpoint']
cmd += ' --feedback-pubsub-endpoint %s' % \
options['feedback_publisher']
else:
cmd = run_bench
try:
# creating a virtualenv there
run_func(queue, job_id, 'virtualenv --no-site-packages .')
run_func(queue, job_id, run_pip + ' install funkload')
# install dependencies if any
for dep in deps:
run_func(queue, job_id, run_pip + ' install %s' % dep)
if fl_result_path is not None:
# in funkload this is a relative path
target = os.path.join(target, fl_result_path)
xml_files = os.path.join(target, '*.xml')
if cycles is None:
cycles = config.get('cycles')
if cycles is not None:
cmd += ' --cycles=%s' % cycles
if duration is None:
duration = config.get('duration')
if test is None:
test = config.get('test')
if script is None:
script = config.get('script')
if duration is not None:
cmd += ' --duration=%s' % duration
report_dir = os.path.join(reportsdir,
os.environ.get('MARTEAU_JOBID', 'report'))
if fixture_plugin:
_logrun('Running the %r fixture' % fixture_plugin)
fixture_klass = get_fixture(fixture_plugin)
if fixture_options is None:
fixture_options = {}
try:
fixture = fixture_klass(**fixture_options)
except:
_logrun('Could not instanciate a fixture plugin instance')
raise
try:
fixture.setup()
except:
_logrun('The fixture set up failed')
raise
# starting the feedback subscriberin its own thread
if rtfeedback:
from funkload.rtfeedback import FeedbackSubscriber
sub = FeedbackSubscriber(pubsub_endpoint=options['feedback_publisher'],
handler=_rt_handler,
pid=os.getpid(), queue=queue)
sub.start()
try:
_logrun('Running the loadtest')
run_func(queue, job_id, '%s %s %s' % (cmd, script, test))
finally:
_logrun('Running the fixture tear_down method')
if fixture_plugin:
try:
fixture.tear_down()
except:
_logrun('The fixture tear down failed')
raise
if rtfeedback:
sub.terminate()
_logrun('Building the report')
report = run_report + ' --skip-definitions --css %s --html -r %s %s'
run_func(queue, job_id, report % (CSS_FILE, report_dir, xml_files))
# do we send an email with the result ?
if email is None:
email = config.get('email')
if email is not None:
_logrun('Sending an e-mail to %r' % email)
try:
res, msg = send_report(email, job_id, **options)
except Exception, e:
res = False
msg = str(e)
if not res:
_logrun(msg)
else:
_logrun('Mail sent.')
return report_dir
def main():
parser = argparse.ArgumentParser(description='Marteau Server')
parser.add_argument('config', help='Config file', nargs='?')
parser.add_argument('--version',
action='store_true',
default=False,
help='Displays Marteau version and exits.')
parser.add_argument('--log-level',
dest='loglevel',
default='info',
choices=LOG_LEVELS.keys() +
[key.upper() for key in LOG_LEVELS.keys()],
help="log level")
parser.add_argument('--log-output',
dest='logoutput',
default='-',
help="log output")
parser.add_argument('--host', help='Host', default='0.0.0.0')
parser.add_argument('--port', help='Port', type=int, default=8080)
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
# configure the logger
configure_logger(logger, args.loglevel, args.logoutput)
# loading the config file
config = Config(args.config)
# loading the app & the queue
global_config = {}
if config.has_section('marteau'):
settings = config.get_map('marteau')
else:
settings = {}
# check is redis is running
if not redis_available():
raise IOError('Marteau needs Redis to run.')
# loading the fixtures plugins
for fixture in settings.get('fixtures', []):
import_string(fixture)
logger.info('Loaded plugins: %s' % ', '.join(get_fixtures()))
app = webapp(global_config, **settings)
try:
httpd = SocketIOServer((args.host, args.port),
app,
resource="socket.io",
policy_server=False)
logger.info('Hammer ready, at http://%s:%s. Where are the nails ?' %
(args.host, args.port))
httpd.serve_forever()
except KeyboardInterrupt:
sys.exit(0)
finally:
logger.info('Bye!')