def ec2_up(db):
log('ec2_up on ' + db)
include_management_node = get_db(db)['has_management_node']
dbs = [('DB', host_counts['DB']), ('YCSB', host_counts['YCSB'])]
if include_management_node:
dbs += [('DB_MAN', 1)]
log('booting nodes for tags %s' % ", ".join([x[0] for x in dbs]))
# Start eveything up first (as slow)
for tag in dbs:
start(tag[0], tag[1])
# Configure
for tag in dbs:
configure_machine_and_reboot(tag[0], tag[1])
refresh_hosts_cache()
# Check started
print emphasis(
"waiting for hosts to start after reboot - ctr-c if you don't want to wait"
)
for tag in dbs:
helper.wait_for_tagged_hosts_to_start(tag[0], tag[1])
def clean_logs(force=False, db=None):
"""Removes all logs from /dev/shm"""
home = '/dev/shm'
if db:
database = get_db(db)
home = database['home']
if force or confirm(red("Do you want to clear all logs from RAM?")):
run('rm -rf %s/*' % home)
def clean_logs(force=False, db=None):
"""Removes all logs from /run/shm"""
home = '/run/shm'
if db:
database = get_db(db)
home = database['home']
if force or confirm(red("Do you want to clear all logs from RAM?")):
run('rm -rf %s/*' % home)
def call(self):
database = get_db(self.db)
the_time = self.time
def inner():
command = _at(prepare_startcmd(database), the_time)
run(command, shell=True)
with almost_nothing():
tasks.execute(inner, hosts=self.hosts)
print green("at %s submitted server start (%s)" % (self.time, self.hosts))
def call(self):
database = get_db(self.db)
the_time = self.time
def inner():
with cd(self.dir_name):
command = _at(prepare_killcmd(database), the_time)
run(command)
with almost_nothing():
tasks.execute(inner, hosts=self.hosts)
print "at %s submitted server kill (%s)" % (self.time, self.hosts)
def initialize_servers(db):
"""
Prepares server hosts to run the failover test
"""
database = get_db(db)
local_dir = os.path.dirname(__file__)
def inner_initialize():
for file in database['failover']['files']:
put(os.path.join(local_dir, file), file, mode=0744)
print green('host %s initialized ' % hosts.env.host)
with almost_nothing():
tasks.execute(inner_initialize, hosts=servers)
def run_workload(db, workload, target=None):
"""Starts running of the workload"""
timestamp = base_time()
print green(timestamp, bold = True)
database = get_db(db)
load = get_workload(workload)
with cd(database['home']):
if target is not None:
part = int(target) / len(env.roledefs['client'])
run(_at(_ycsbruncmd(database, load, timestamp, part), timestamp))
else:
run(_at(_ycsbruncmd(database, load, timestamp), timestamp))
def call(self):
database = get_db(self.db)
the_time = self.time
def inner():
command = _at(prepare_startcmd(database), the_time)
run(command, shell=True)
with almost_nothing():
tasks.execute(inner, hosts=self.hosts)
print green("at %s submitted server start (%s)" %
(self.time, self.hosts))
def run_workload(db, workload, target=None):
"""Starts running of the workload"""
timestamp = base_time()
print green(timestamp, bold=True)
database = get_db(db)
load = get_workload(workload)
with cd(database['home']):
if target is not None:
part = int(target) / len(env.roledefs['client'])
run(_at(_ycsbruncmd(database, load, timestamp, part), timestamp))
else:
run(_at(_ycsbruncmd(database, load, timestamp), timestamp))
def load(db, target=None):
"""Starts loading of data to the database"""
timestamp = base_time()
print green(timestamp, bold = True)
clientno = _client_no()
database = get_db(db)
with cd(database['home']):
if target is not None:
part = int(target) / len(env.roledefs['client'])
run(_at(_ycsbloadcmd(database, clientno, timestamp, part), timestamp))
else:
run(_at(_ycsbloadcmd(database, clientno, timestamp), timestamp))
def initialize_servers(db):
"""
Prepares server hosts to run the failover test
"""
database = get_db(db)
local_dir = os.path.dirname(__file__)
def inner_initialize():
for file in database['failover']['files']:
put(os.path.join(local_dir, file), file, mode=0744)
print green('host %s initialized ' % hosts.env.host)
with almost_nothing():
tasks.execute(inner_initialize, hosts=servers)
def status(db):
""" Shows status of the currently running YCSBs """
with almost_nothing():
database = get_db(db)
dir_name = database['home']
with cd(database['home']):
# we need to provide the info for the latest modified
# file, therefore we need to check the times
# recursively list all err files and modification times
lines = run('find . -name "*.err" -printf "%p %T@\n"').split(
'\r\n')
def extract(line):
(f, t) = line.split(' ')
(d, f) = os.path.split(f)
return (d, f, float(t))
def order(t):
return -t[2]
if lines and not (len(lines) == 1 and not lines[0]):
files = sorted(map(extract, lines), key=order)
(d, f, t) = files[0]
dir_name = os.path.normpath(os.path.join(database['home'], d))
with cd(dir_name):
ls_out = run('ls --format=single-column --sort=t *.lock')
msg = green('free') if 'cannot access' in ls_out else red('locked')
print blue('Lock:', bold=True), msg
print blue('Scheduled:', bold=True)
# print run('tail -n 2 /var/spool/cron/atjobs/*')
print sudo('atq')
print blue('Running:', bold=True)
print run('ps -f -C java')
# print run('ps aux | grep python')
# sort the output of ls by date, the first entry should be the *.err needed
ls_out = run('ls --format=single-column --sort=t *.err')
if 'cannot access' in ls_out:
logfile = '<no any *.err files>'
tail = ''
else:
ls = ls_out.split("\r\n")
logfile = ls[0]
tail = run('tail %s' % logfile)
print blue('Dir:', bold=True), green(dir_name, bold=True)
print blue('Log:', bold=True), green(logfile, bold=True)
print tail
# print blue('List:', bold = True)
# ls_out = run('ls --format=single-column --sort=t')
# print ls_out
print
def load(db, target=None):
"""Starts loading of data to the database"""
timestamp = base_time()
print green(timestamp, bold=True)
clientno = _client_no()
database = get_db(db)
with cd(database['home']):
if target is not None:
part = int(target) / len(env.roledefs['client'])
run(
_at(_ycsbloadcmd(database, clientno, timestamp, part),
timestamp))
else:
run(_at(_ycsbloadcmd(database, clientno, timestamp), timestamp))
def db_up(db, force=False):
force = bool(force)
log('db_up: '+db)
actions = get_db(db)['actions']
if force or not amazon_helper.stage_complete("DB","db-install"):
actions['install']()
log('Installing %s' % db)
amazon_helper.record_stage("DB","db-install")
else :
print 'Skipping install of '+db
log('Starting %s' % db)
actions['run']()
def submit_workload(the_hosts, dir_name, db, workload, the_time, target = None):
"""
Schedules the workload.
Note: we cannot use ycsb.workload, because it is decorated
"""
database = get_db(db)
load = get_workload(workload)
def inner_submit_workload():
with cd(dir_name):
param = int(target) / len(the_hosts) if target is not None else None
# command = prepare_ycsbruncmd(database, load, the_time, param)
command = _at(prepare_ycsbruncmd(the_hosts, dir_name, database, load, the_time, param), the_time)
run(command)
with almost_nothing():
tasks.execute(inner_submit_workload, hosts=the_hosts)
def status(db):
""" Shows status of the currently running YCSBs """
with almost_nothing():
database = get_db(db)
dir_name = database['home']
with cd(database['home']):
# we need to provide the info for the latest modified
# file, therefore we need to check the times
# recursively list all err files and modification times
lines = run('find . -name "*.err" -printf "%p %T@\n"').split('\r\n')
def extract(line):
(f, t) = line.split(' ')
(d, f) = os.path.split(f)
return (d, f, float(t))
def order(t):
return -t[2]
if lines and not (len(lines) == 1 and not lines[0]):
files = sorted(map(extract, lines), key = order)
(d, f, t) = files[0]
dir_name = os.path.normpath(os.path.join(database['home'], d))
with cd(dir_name):
ls_out = run('ls --format=single-column --sort=t *.lock')
msg = green('free') if 'cannot access' in ls_out else red('locked')
print blue('Lock:', bold = True), msg
print blue('Scheduled:', bold = True)
# print run('tail -n 2 /var/spool/cron/atjobs/*')
print sudo('atq')
print blue('Running:', bold = True)
print run('ps -f -C java')
# print run('ps aux | grep python')
# sort the output of ls by date, the first entry should be the *.err needed
ls_out = run('ls --format=single-column --sort=t *.err')
if 'cannot access' in ls_out:
logfile = '<no any *.err files>'
tail = ''
else:
ls = ls_out.split("\r\n")
logfile = ls[0]
tail = run('tail %s' % logfile)
print blue('Dir:', bold = True), green(dir_name, bold = True)
print blue('Log:', bold = True), green(logfile, bold = True)
print tail
# print blue('List:', bold = True)
# ls_out = run('ls --format=single-column --sort=t')
# print ls_out
print
def submit_workload(the_hosts, dir_name, db, workload, the_time, target=None):
"""
Schedules the workload.
Note: we cannot use ycsb.workload, because it is decorated
"""
database = get_db(db)
load = get_workload(workload)
def inner_submit_workload():
with cd(dir_name):
param = int(target) / len(
the_hosts) if target is not None else None
# command = prepare_ycsbruncmd(database, load, the_time, param)
command = _at(
prepare_ycsbruncmd(the_hosts, dir_name, database, load,
the_time, param), the_time)
run(command)
with almost_nothing():
tasks.execute(inner_submit_workload, hosts=the_hosts)
def test():
get_db('mongodb')['actions']['run']()
def get_log(db, regex='.*', do=False):
""" Download *.err and *.out logs satisfying the regex to be transferred
OR transfer all logs in the batch dir
"""
database = get_db(db)
with almost_nothing():
cn = _client_no() + 1
with cd(database['home']):
(f0, is_dir) = determine_file(regex)
print blue('Filename at c%s: ' % cn, bold = True), green(f0, bold = True)
# now do the processing, if enabled
#If do is presented, then do is str. bool(str) = True if str is not empty
#May be we should process str as bool?
if do:
with cd(database['home']):
if is_dir:
tempdir_local = '%s/c%s' % (tempfile.gettempdir(), cn)
dir_local = './%s-c%s' % (f0, cn)
bz2_remote = '%s-c%s-dir.bz2' % (f0, cn)
bz2_full_local = '%s/%s-dir.bz2' % (tempdir_local, f0)
# packing
print blue('c%s packing ...' % cn)
#run('tar -jcvf %s %s' % (bz2_remote, f0))
#To ignore huge err files.
run('tar -jcvf %s %s/*.out' % (bz2_remote, f0))
# download them
print blue('c%s transferring to %s...' % (cn, tempdir_local))
get(bz2_remote, bz2_full_local)
# the files are here, remove remote bz2
run('rm -f %s' % bz2_remote)
# unpacking to temp dir
print blue('c%s unpacking ...' % cn)
local('tar -xvf %s -C %s' % (bz2_full_local, tempdir_local))
print blue('c%s moving to current dir ...' % cn)
# remove the old version of the dir
# TODO maybe use versioning?
local('rm -rf %s' % dir_local)
local('mkdir -p %s' % dir_local)
local('mv %s/%s/* %s' % (tempdir_local, f0, dir_local))
# additional step - rename all the files
# *.err and *.out in the folder
rename_cmd = 'for i in ./%s/*.*; do ' \
' ext="${i##*.}" ; ' \
' fil="${i%%.*}-c%s" ; ' \
' mv "$i" "$fil.$ext" ; ' \
'done' % (dir_local, cn)
local(rename_cmd)
local('rm -rf %s' % tempdir_local)
else:
tempdir_local = '%s/c%s' % (tempfile.gettempdir(), cn)
bz2err_remote = '%s-c%s-err.bz2' % (f0, cn)
bz2out_remote = '%s-c%s-out.bz2' % (f0, cn)
bz2err_full_local = '%s/%s-err.bz2' % (tempdir_local, f0)
bz2out_full_local = '%s/%s-out.bz2' % (tempdir_local, f0)
# packing
print blue('c%s packing ...' % cn)
run('tar -jcvf %s %s.err' % (bz2err_remote, f0))
run('tar -jcvf %s %s.out' % (bz2out_remote, f0))
# download them
print blue('c%s transferring to %s...' % (cn, tempdir_local))
get(bz2err_remote, bz2err_full_local)
get(bz2out_remote, bz2out_full_local)
# the files are here, remove remote bz2
run('rm -f %s' % bz2err_remote)
run('rm -f %s' % bz2out_remote)
# unpacking to temp dir
print blue('c%s unpacking ...' % cn)
local('tar -xvf %s -C %s' % (bz2err_full_local, tempdir_local))
local('tar -xvf %s -C %s' % (bz2out_full_local, tempdir_local))
# unpacked ok, remove local bz2
#local('rm -f %s' % bz2err_full_local)
#local('rm -f %s' % bz2out_full_local)
print blue('c%s moving to current dir ...' % cn)
local('mv %s/%s.err ./%s-c%s.err' % (tempdir_local, f0, f0, cn))
local('mv %s/%s.out ./%s-c%s.out' % (tempdir_local, f0, f0, cn))
local('rm -rf %s' % tempdir_local)
def initialize(the_hosts, db):
"""
Prepares hosts to run the series
"""
database = get_db(db)
db_home = database['home']
pf = compile('^%s' % database['name'])
pn = compile('(\d+)/$')
nos = [0]
def inner_initialize_0():
# sudo('yum -y install at')
# sudo('service atd start')
# sudo('sudo yum install -y java-1.7.0-openjdk-devel')
# with cd('/opt'):
# put('../distribution/target/ycsb-0.1.4.tar.gz', '/run/shm/ycsb.tar.gz')
# sudo('rm -r ycsb-0.1.4')
# sudo('tar xzvf /run/shm/ycsb.tar.gz')
# sudo('ln -s /opt/ycsb-0.1.4 /opt/ycsb')
# print 'ycsb deployed'
sudo('mkdir -p %s ; chmod 1777 %s' % (db_home, db_home))
with cd(db_home):
ls = run("ls --format=single-column --sort=t -d -- */")
# the most recent file satisfying pattern
if ls:
ls = ls.split('\r\n')
file_names = [f for f in ls if pf.search(f)]
for file_name in file_names:
mn = pn.search(file_name)
if mn:
nos.append(int(mn.group(1)) + 1)
# find the maximum number for all of the hosts
with almost_nothing():
tasks.execute(inner_initialize_0, hosts=the_hosts)
# now form the dir name
dir_name = os.path.join(database['home'],
'%s_%02d' % (database['name'], max(nos)))
def inner_initialize_1():
run('mkdir %s ' % dir_name)
series_dir = os.path.dirname(__file__)
local_benchmark_script = os.path.join(series_dir, benchmark_script)
if LOCAL:
with cd(dir_name):
run('rm -rf ./*')
put(local_benchmark_script, benchmark_script, mode=0744)
# run('sed -i "s/\/opt\/ycsb\/bin\/ycsb \$\*/python nbody.py \$\*/g" %s' % benchmark_script)
else:
# if not LOCAL
with cd(dir_name):
put(local_benchmark_script, benchmark_script, mode=0744)
# continue init
# clear all the tasks that submitted so far
with cd(dir_name):
tasks = run('atq').split('\r\n')
tid = []
for task in tasks:
m = search('^(\d+)\t', task)
if m:
tid.append(m.group(1))
if tid:
run('atrm %s' % ' '.join(tid))
print green('host %s initialized ' % hosts.env.host)
with almost_nothing():
tasks.execute(inner_initialize_1, hosts=the_hosts)
return dir_name
def db_status(db):
"""Shows the status of the DB"""
database = get_db(db)
with settings(hide('stdout'), hosts=database['status']['hosts']):
print run(database['status']['command'])
def db_down(db):
log('db_down: '+db)
get_db(db)['actions']['stop']()
def initialize(the_hosts, db):
"""
Prepares hosts to run the series
"""
database = get_db(db)
db_home = database['home']
pf = compile('^%s' % database['name'])
pn = compile('(\d+)/$')
nos = [0]
def inner_initialize_0():
# sudo('yum -y install at')
# sudo('service atd start')
# sudo('sudo yum install -y java-1.7.0-openjdk-devel')
# with cd('/opt'):
# put('../distribution/target/ycsb-0.1.4.tar.gz', '/run/shm/ycsb.tar.gz')
# sudo('rm -r ycsb-0.1.4')
# sudo('tar xzvf /run/shm/ycsb.tar.gz')
# sudo('ln -s /opt/ycsb-0.1.4 /opt/ycsb')
# print 'ycsb deployed'
sudo('mkdir -p %s ; chmod 1777 %s' % (db_home, db_home))
with cd(db_home):
ls = run("ls --format=single-column --sort=t -d -- */")
# the most recent file satisfying pattern
if ls:
ls = ls.split('\r\n')
file_names = [f for f in ls if pf.search(f)]
for file_name in file_names:
mn = pn.search(file_name)
if mn:
nos.append(int(mn.group(1)) + 1)
# find the maximum number for all of the hosts
with almost_nothing():
tasks.execute(inner_initialize_0, hosts=the_hosts)
# now form the dir name
dir_name = os.path.join(database['home'], '%s_%02d' % (database['name'], max(nos)))
def inner_initialize_1():
run('mkdir %s ' % dir_name)
series_dir = os.path.dirname(__file__)
local_benchmark_script = os.path.join(series_dir, benchmark_script)
if LOCAL:
with cd(dir_name):
run('rm -rf ./*')
put(local_benchmark_script, benchmark_script, mode=0744)
# run('sed -i "s/\/opt\/ycsb\/bin\/ycsb \$\*/python nbody.py \$\*/g" %s' % benchmark_script)
else:
# if not LOCAL
with cd(dir_name):
put(local_benchmark_script, benchmark_script, mode=0744)
# continue init
# clear all the tasks that submitted so far
with cd(dir_name):
tasks = run('atq').split('\r\n')
tid = []
for task in tasks:
m = search('^(\d+)\t', task)
if m:
tid.append(m.group(1))
if tid:
run('atrm %s' % ' '.join(tid))
print green('host %s initialized ' % hosts.env.host)
with almost_nothing():
tasks.execute(inner_initialize_1, hosts=the_hosts)
return dir_name
def get_log(db, regex='.*', do=False):
""" Download *.err and *.out logs satisfying the regex to be transferred
OR transfer all logs in the batch dir
"""
database = get_db(db)
with almost_nothing():
cn = _client_no() + 1
with cd(database['home']):
(f0, is_dir) = determine_file(regex)
print blue('Filename at c%s: ' % cn, bold=True), green(f0,
bold=True)
# now do the processing, if enabled
#If do is presented, then do is str. bool(str) = True if str is not empty
#May be we should process str as bool?
if do:
with cd(database['home']):
if is_dir:
tempdir_local = '%s/c%s' % (tempfile.gettempdir(), cn)
dir_local = './%s-c%s' % (f0, cn)
bz2_remote = '%s-c%s-dir.bz2' % (f0, cn)
bz2_full_local = '%s/%s-dir.bz2' % (tempdir_local, f0)
# packing
print blue('c%s packing ...' % cn)
#run('tar -jcvf %s %s' % (bz2_remote, f0))
#To ignore huge err files.
run('tar -jcvf %s %s/*.out' % (bz2_remote, f0))
# download them
print blue('c%s transferring to %s...' %
(cn, tempdir_local))
get(bz2_remote, bz2_full_local)
# the files are here, remove remote bz2
run('rm -f %s' % bz2_remote)
# unpacking to temp dir
print blue('c%s unpacking ...' % cn)
local('tar -xvf %s -C %s' %
(bz2_full_local, tempdir_local))
print blue('c%s moving to current dir ...' % cn)
# remove the old version of the dir
# TODO maybe use versioning?
local('rm -rf %s' % dir_local)
local('mkdir -p %s' % dir_local)
local('mv %s/%s/* %s' % (tempdir_local, f0, dir_local))
# additional step - rename all the files
# *.err and *.out in the folder
rename_cmd = 'for i in ./%s/*.*; do ' \
' ext="${i##*.}" ; ' \
' fil="${i%%.*}-c%s" ; ' \
' mv "$i" "$fil.$ext" ; ' \
'done' % (dir_local, cn)
local(rename_cmd)
local('rm -rf %s' % tempdir_local)
else:
tempdir_local = '%s/c%s' % (tempfile.gettempdir(), cn)
bz2err_remote = '%s-c%s-err.bz2' % (f0, cn)
bz2out_remote = '%s-c%s-out.bz2' % (f0, cn)
bz2err_full_local = '%s/%s-err.bz2' % (tempdir_local, f0)
bz2out_full_local = '%s/%s-out.bz2' % (tempdir_local, f0)
# packing
print blue('c%s packing ...' % cn)
run('tar -jcvf %s %s.err' % (bz2err_remote, f0))
run('tar -jcvf %s %s.out' % (bz2out_remote, f0))
# download them
print blue('c%s transferring to %s...' %
(cn, tempdir_local))
get(bz2err_remote, bz2err_full_local)
get(bz2out_remote, bz2out_full_local)
# the files are here, remove remote bz2
run('rm -f %s' % bz2err_remote)
run('rm -f %s' % bz2out_remote)
# unpacking to temp dir
print blue('c%s unpacking ...' % cn)
local('tar -xvf %s -C %s' %
(bz2err_full_local, tempdir_local))
local('tar -xvf %s -C %s' %
(bz2out_full_local, tempdir_local))
# unpacked ok, remove local bz2
#local('rm -f %s' % bz2err_full_local)
#local('rm -f %s' % bz2out_full_local)
print blue('c%s moving to current dir ...' % cn)
local('mv %s/%s.err ./%s-c%s.err' %
(tempdir_local, f0, f0, cn))
local('mv %s/%s.out ./%s-c%s.out' %
(tempdir_local, f0, f0, cn))
local('rm -rf %s' % tempdir_local)
