def call(self): database = get_db(self.db) the_time = self.time def inner(): command = _at(prepare_startcmd(database), the_time) run(command, shell=True) with almost_nothing(): tasks.execute(inner, hosts=self.hosts) print green("at %s submitted server start (%s)" % (self.time, self.hosts))
def initialize_servers(db): """ Prepares server hosts to run the failover test """ database = get_db(db) local_dir = os.path.dirname(__file__) def inner_initialize(): for file in database['failover']['files']: put(os.path.join(local_dir, file), file, mode=0744) print green('host %s initialized ' % hosts.env.host) with almost_nothing(): tasks.execute(inner_initialize, hosts=servers)
def call(self): database = get_db(self.db) the_time = self.time def inner(): with cd(self.dir_name): command = _at(prepare_killcmd(database), the_time) run(command) with almost_nothing(): tasks.execute(inner, hosts=self.hosts) print "at %s submitted server kill (%s)" % (self.time, self.hosts)
def status(db): """ Shows status of the currently running YCSBs """ with almost_nothing(): database = get_db(db) dir_name = database['home'] with cd(database['home']): # we need to provide the info for the latest modified # file, therefore we need to check the times # recursively list all err files and modification times lines = run('find . -name "*.err" -printf "%p %T@\n"').split( '\r\n') def extract(line): (f, t) = line.split(' ') (d, f) = os.path.split(f) return (d, f, float(t)) def order(t): return -t[2] if lines and not (len(lines) == 1 and not lines[0]): files = sorted(map(extract, lines), key=order) (d, f, t) = files[0] dir_name = os.path.normpath(os.path.join(database['home'], d)) with cd(dir_name): ls_out = run('ls --format=single-column --sort=t *.lock') msg = green('free') if 'cannot access' in ls_out else red('locked') print blue('Lock:', bold=True), msg print blue('Scheduled:', bold=True) # print run('tail -n 2 /var/spool/cron/atjobs/*') print sudo('atq') print blue('Running:', bold=True) print run('ps -f -C java') # print run('ps aux | grep python') # sort the output of ls by date, the first entry should be the *.err needed ls_out = run('ls --format=single-column --sort=t *.err') if 'cannot access' in ls_out: logfile = '<no any *.err files>' tail = '' else: ls = ls_out.split("\r\n") logfile = ls[0] tail = run('tail %s' % logfile) print blue('Dir:', bold=True), green(dir_name, bold=True) print blue('Log:', bold=True), green(logfile, bold=True) print tail # print blue('List:', bold = True) # ls_out = run('ls --format=single-column --sort=t') # print ls_out print
def submit_workload(the_hosts, dir_name, db, workload, the_time, target = None): """ Schedules the workload. Note: we cannot use ycsb.workload, because it is decorated """ database = get_db(db) load = get_workload(workload) def inner_submit_workload(): with cd(dir_name): param = int(target) / len(the_hosts) if target is not None else None # command = prepare_ycsbruncmd(database, load, the_time, param) command = _at(prepare_ycsbruncmd(the_hosts, dir_name, database, load, the_time, param), the_time) run(command) with almost_nothing(): tasks.execute(inner_submit_workload, hosts=the_hosts)
def status(db): """ Shows status of the currently running YCSBs """ with almost_nothing(): database = get_db(db) dir_name = database['home'] with cd(database['home']): # we need to provide the info for the latest modified # file, therefore we need to check the times # recursively list all err files and modification times lines = run('find . -name "*.err" -printf "%p %T@\n"').split('\r\n') def extract(line): (f, t) = line.split(' ') (d, f) = os.path.split(f) return (d, f, float(t)) def order(t): return -t[2] if lines and not (len(lines) == 1 and not lines[0]): files = sorted(map(extract, lines), key = order) (d, f, t) = files[0] dir_name = os.path.normpath(os.path.join(database['home'], d)) with cd(dir_name): ls_out = run('ls --format=single-column --sort=t *.lock') msg = green('free') if 'cannot access' in ls_out else red('locked') print blue('Lock:', bold = True), msg print blue('Scheduled:', bold = True) # print run('tail -n 2 /var/spool/cron/atjobs/*') print sudo('atq') print blue('Running:', bold = True) print run('ps -f -C java') # print run('ps aux | grep python') # sort the output of ls by date, the first entry should be the *.err needed ls_out = run('ls --format=single-column --sort=t *.err') if 'cannot access' in ls_out: logfile = '<no any *.err files>' tail = '' else: ls = ls_out.split("\r\n") logfile = ls[0] tail = run('tail %s' % logfile) print blue('Dir:', bold = True), green(dir_name, bold = True) print blue('Log:', bold = True), green(logfile, bold = True) print tail # print blue('List:', bold = True) # ls_out = run('ls --format=single-column --sort=t') # print ls_out print
def submit_workload(the_hosts, dir_name, db, workload, the_time, target=None): """ Schedules the workload. Note: we cannot use ycsb.workload, because it is decorated """ database = get_db(db) load = get_workload(workload) def inner_submit_workload(): with cd(dir_name): param = int(target) / len( the_hosts) if target is not None else None # command = prepare_ycsbruncmd(database, load, the_time, param) command = _at( prepare_ycsbruncmd(the_hosts, dir_name, database, load, the_time, param), the_time) run(command) with almost_nothing(): tasks.execute(inner_submit_workload, hosts=the_hosts)
def get_log(db, regex='.*', do=False): """ Download *.err and *.out logs satisfying the regex to be transferred OR transfer all logs in the batch dir """ database = get_db(db) with almost_nothing(): cn = _client_no() + 1 with cd(database['home']): (f0, is_dir) = determine_file(regex) print blue('Filename at c%s: ' % cn, bold = True), green(f0, bold = True) # now do the processing, if enabled #If do is presented, then do is str. bool(str) = True if str is not empty #May be we should process str as bool? if do: with cd(database['home']): if is_dir: tempdir_local = '%s/c%s' % (tempfile.gettempdir(), cn) dir_local = './%s-c%s' % (f0, cn) bz2_remote = '%s-c%s-dir.bz2' % (f0, cn) bz2_full_local = '%s/%s-dir.bz2' % (tempdir_local, f0) # packing print blue('c%s packing ...' % cn) #run('tar -jcvf %s %s' % (bz2_remote, f0)) #To ignore huge err files. run('tar -jcvf %s %s/*.out' % (bz2_remote, f0)) # download them print blue('c%s transferring to %s...' % (cn, tempdir_local)) get(bz2_remote, bz2_full_local) # the files are here, remove remote bz2 run('rm -f %s' % bz2_remote) # unpacking to temp dir print blue('c%s unpacking ...' % cn) local('tar -xvf %s -C %s' % (bz2_full_local, tempdir_local)) print blue('c%s moving to current dir ...' % cn) # remove the old version of the dir # TODO maybe use versioning? local('rm -rf %s' % dir_local) local('mkdir -p %s' % dir_local) local('mv %s/%s/* %s' % (tempdir_local, f0, dir_local)) # additional step - rename all the files # *.err and *.out in the folder rename_cmd = 'for i in ./%s/*.*; do ' \ ' ext="${i##*.}" ; ' \ ' fil="${i%%.*}-c%s" ; ' \ ' mv "$i" "$fil.$ext" ; ' \ 'done' % (dir_local, cn) local(rename_cmd) local('rm -rf %s' % tempdir_local) else: tempdir_local = '%s/c%s' % (tempfile.gettempdir(), cn) bz2err_remote = '%s-c%s-err.bz2' % (f0, cn) bz2out_remote = '%s-c%s-out.bz2' % (f0, cn) bz2err_full_local = '%s/%s-err.bz2' % (tempdir_local, f0) bz2out_full_local = '%s/%s-out.bz2' % (tempdir_local, f0) # packing print blue('c%s packing ...' % cn) run('tar -jcvf %s %s.err' % (bz2err_remote, f0)) run('tar -jcvf %s %s.out' % (bz2out_remote, f0)) # download them print blue('c%s transferring to %s...' % (cn, tempdir_local)) get(bz2err_remote, bz2err_full_local) get(bz2out_remote, bz2out_full_local) # the files are here, remove remote bz2 run('rm -f %s' % bz2err_remote) run('rm -f %s' % bz2out_remote) # unpacking to temp dir print blue('c%s unpacking ...' % cn) local('tar -xvf %s -C %s' % (bz2err_full_local, tempdir_local)) local('tar -xvf %s -C %s' % (bz2out_full_local, tempdir_local)) # unpacked ok, remove local bz2 #local('rm -f %s' % bz2err_full_local) #local('rm -f %s' % bz2out_full_local) print blue('c%s moving to current dir ...' % cn) local('mv %s/%s.err ./%s-c%s.err' % (tempdir_local, f0, f0, cn)) local('mv %s/%s.out ./%s-c%s.out' % (tempdir_local, f0, f0, cn)) local('rm -rf %s' % tempdir_local)
def initialize(the_hosts, db): """ Prepares hosts to run the series """ database = get_db(db) db_home = database['home'] pf = compile('^%s' % database['name']) pn = compile('(\d+)/$') nos = [0] def inner_initialize_0(): # sudo('yum -y install at') # sudo('service atd start') # sudo('sudo yum install -y java-1.7.0-openjdk-devel') # with cd('/opt'): # put('../distribution/target/ycsb-0.1.4.tar.gz', '/run/shm/ycsb.tar.gz') # sudo('rm -r ycsb-0.1.4') # sudo('tar xzvf /run/shm/ycsb.tar.gz') # sudo('ln -s /opt/ycsb-0.1.4 /opt/ycsb') # print 'ycsb deployed' sudo('mkdir -p %s ; chmod 1777 %s' % (db_home, db_home)) with cd(db_home): ls = run("ls --format=single-column --sort=t -d -- */") # the most recent file satisfying pattern if ls: ls = ls.split('\r\n') file_names = [f for f in ls if pf.search(f)] for file_name in file_names: mn = pn.search(file_name) if mn: nos.append(int(mn.group(1)) + 1) # find the maximum number for all of the hosts with almost_nothing(): tasks.execute(inner_initialize_0, hosts=the_hosts) # now form the dir name dir_name = os.path.join(database['home'], '%s_%02d' % (database['name'], max(nos))) def inner_initialize_1(): run('mkdir %s ' % dir_name) series_dir = os.path.dirname(__file__) local_benchmark_script = os.path.join(series_dir, benchmark_script) if LOCAL: with cd(dir_name): run('rm -rf ./*') put(local_benchmark_script, benchmark_script, mode=0744) # run('sed -i "s/\/opt\/ycsb\/bin\/ycsb \$\*/python nbody.py \$\*/g" %s' % benchmark_script) else: # if not LOCAL with cd(dir_name): put(local_benchmark_script, benchmark_script, mode=0744) # continue init # clear all the tasks that submitted so far with cd(dir_name): tasks = run('atq').split('\r\n') tid = [] for task in tasks: m = search('^(\d+)\t', task) if m: tid.append(m.group(1)) if tid: run('atrm %s' % ' '.join(tid)) print green('host %s initialized ' % hosts.env.host) with almost_nothing(): tasks.execute(inner_initialize_1, hosts=the_hosts) return dir_name
def get_log(db, regex='.*', do=False): """ Download *.err and *.out logs satisfying the regex to be transferred OR transfer all logs in the batch dir """ database = get_db(db) with almost_nothing(): cn = _client_no() + 1 with cd(database['home']): (f0, is_dir) = determine_file(regex) print blue('Filename at c%s: ' % cn, bold=True), green(f0, bold=True) # now do the processing, if enabled #If do is presented, then do is str. bool(str) = True if str is not empty #May be we should process str as bool? if do: with cd(database['home']): if is_dir: tempdir_local = '%s/c%s' % (tempfile.gettempdir(), cn) dir_local = './%s-c%s' % (f0, cn) bz2_remote = '%s-c%s-dir.bz2' % (f0, cn) bz2_full_local = '%s/%s-dir.bz2' % (tempdir_local, f0) # packing print blue('c%s packing ...' % cn) #run('tar -jcvf %s %s' % (bz2_remote, f0)) #To ignore huge err files. run('tar -jcvf %s %s/*.out' % (bz2_remote, f0)) # download them print blue('c%s transferring to %s...' % (cn, tempdir_local)) get(bz2_remote, bz2_full_local) # the files are here, remove remote bz2 run('rm -f %s' % bz2_remote) # unpacking to temp dir print blue('c%s unpacking ...' % cn) local('tar -xvf %s -C %s' % (bz2_full_local, tempdir_local)) print blue('c%s moving to current dir ...' % cn) # remove the old version of the dir # TODO maybe use versioning? local('rm -rf %s' % dir_local) local('mkdir -p %s' % dir_local) local('mv %s/%s/* %s' % (tempdir_local, f0, dir_local)) # additional step - rename all the files # *.err and *.out in the folder rename_cmd = 'for i in ./%s/*.*; do ' \ ' ext="${i##*.}" ; ' \ ' fil="${i%%.*}-c%s" ; ' \ ' mv "$i" "$fil.$ext" ; ' \ 'done' % (dir_local, cn) local(rename_cmd) local('rm -rf %s' % tempdir_local) else: tempdir_local = '%s/c%s' % (tempfile.gettempdir(), cn) bz2err_remote = '%s-c%s-err.bz2' % (f0, cn) bz2out_remote = '%s-c%s-out.bz2' % (f0, cn) bz2err_full_local = '%s/%s-err.bz2' % (tempdir_local, f0) bz2out_full_local = '%s/%s-out.bz2' % (tempdir_local, f0) # packing print blue('c%s packing ...' % cn) run('tar -jcvf %s %s.err' % (bz2err_remote, f0)) run('tar -jcvf %s %s.out' % (bz2out_remote, f0)) # download them print blue('c%s transferring to %s...' % (cn, tempdir_local)) get(bz2err_remote, bz2err_full_local) get(bz2out_remote, bz2out_full_local) # the files are here, remove remote bz2 run('rm -f %s' % bz2err_remote) run('rm -f %s' % bz2out_remote) # unpacking to temp dir print blue('c%s unpacking ...' % cn) local('tar -xvf %s -C %s' % (bz2err_full_local, tempdir_local)) local('tar -xvf %s -C %s' % (bz2out_full_local, tempdir_local)) # unpacked ok, remove local bz2 #local('rm -f %s' % bz2err_full_local) #local('rm -f %s' % bz2out_full_local) print blue('c%s moving to current dir ...' % cn) local('mv %s/%s.err ./%s-c%s.err' % (tempdir_local, f0, f0, cn)) local('mv %s/%s.out ./%s-c%s.out' % (tempdir_local, f0, f0, cn)) local('rm -rf %s' % tempdir_local)