def tearDown(self):
command = 'deletetable %s -f\n' % self.tablename
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, "Could not delete table")
log.debug("Process finished")
Benchmark.tearDown(self)
def tearDown(self):
command = 'deletetable %s -f\n' % self.tablename
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, "Could not delete table")
log.debug("Process finished")
Benchmark.tearDown(self)
def setUp(self):
random.jumpahead(int(time.time()))
num = random.randint(1, 100000)
self.input_table = self.input_table + "_" + str(num)
self.output_table = self.output_table + "_" + str(num)
#if (not os.getenv("HADOOP_CLASSPATH")):
# os.putenv("HADOOP_CLASSPATH", self.getjars(":"))
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join( dir, 'splits' )
# code, out, err = cloudshell.run(self.username, self.password, 'table RowHashTestInput\n')
# if out.find('no such table') == -1:
# code, out, err = cloudshell.run(self.username, self.password, 'deletetable RowHashTestInput\n')
# self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.input_table, file))
#code, out, err = cloudshell.run('table RowHashTest\n')
#if out.find('no such table') == -1:
# code, out, err = cloudshell.run('user root\nsecret\ndeletetable RowHashTest\n')
# self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.output_table, file))
command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
self.numrows(),
self.keysizemin(),
self.keysizemax(),
self.minvaluesize(),
self.maxvaluesize(),
self.input_table,
self.getInstance(),
self.getZookeepers(),
self.getUsername(),
self.getPassword(),
self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
Benchmark.setUp(self)
def setUp(self):
random.jumpahead(int(time.time()))
num = random.randint(1, 100000)
self.input_table = self.input_table + "_" + str(num)
self.output_table = self.output_table + "_" + str(num)
#if (not os.getenv("HADOOP_CLASSPATH")):
# os.putenv("HADOOP_CLASSPATH", self.getjars(":"))
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join(dir, 'splits')
# code, out, err = cloudshell.run(self.username, self.password, 'table RowHashTestInput\n')
# if out.find('no such table') == -1:
# code, out, err = cloudshell.run(self.username, self.password, 'deletetable RowHashTestInput\n')
# self.sleep(15)
code, out, err = cloudshell.run(
self.username, self.password,
"createtable %s -sf %s\n" % (self.input_table, file))
#code, out, err = cloudshell.run('table RowHashTest\n')
#if out.find('no such table') == -1:
# code, out, err = cloudshell.run('user root\nsecret\ndeletetable RowHashTest\n')
# self.sleep(15)
code, out, err = cloudshell.run(
self.username, self.password,
"createtable %s -sf %s\n" % (self.output_table, file))
command = self.buildcommand(
'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
self.numrows(), self.keysizemin(), self.keysizemax(),
self.minvaluesize(), self.maxvaluesize(), self.input_table,
self.getInstance(), self.getZookeepers(), self.getUsername(),
self.getPassword(), self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
Benchmark.setUp(self)
def setUp(self):
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join( dir, 'splits' )
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.input_table)
if out.find('does not exist') == -1:
code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' % self.input_table)
self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.input_table, file))
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.output_table)
if out.find('does not exist') == -1:
code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' %
self.output_table)
self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.output_table, file))
command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
'--count', self.numrows(),
'-nk', self.keysizemin(),
'-xk', self.keysizemax(),
'-nv', self.minvaluesize(),
'-xv', self.maxvaluesize(),
'--table', self.input_table,
'-i', self.getInstance(),
'-z', self.getZookeepers(),
'-u', self.getUsername(),
'-p', self.getPassword(),
'--splits', self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
Benchmark.setUp(self)
def setUp(self):
# Need to generate a splits file for each speed
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.tablename)
if out.find('does not exist') == -1:
log.debug('Deleting table %s' % self.tablename)
code, out, err = cloudshell.run(self.username, self.password, 'deletetable %s -f\n' % self.tablename)
self.assertEqual(code, 0, "Could not delete table")
Benchmark.setUp(self)
def setUp(self):
# Need to generate a splits file for each speed
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.tablename)
if out.find('does not exist') == -1:
log.debug('Deleting table %s' % self.tablename)
code, out, err = cloudshell.run(self.username, self.password, 'deletetable %s -f\n' % self.tablename)
self.assertEqual(code, 0, "Could not delete table")
Benchmark.setUp(self)
def setUp(self):
code, out, err = cloudshell.run(self.username, self.password,
'table %s\n' % self.tablename)
if out.find('does not exist') == -1:
log.debug('Deleting table %s' % self.tablename)
code, out, err = cloudshell.run(
self.username, self.password,
'deletetable -f %s\n' % self.tablename)
Benchmark.setUp(self)
def setUp(self):
code, out, err = cloudshell.run(self.username, self.password, 'table test_ingest\n')
if out.find('no such table') >= 0:
log.debug("Deleting table test_ingest")
code, out, err = cloudshell.run(self.username, self.password, 'deletetable test_ingest\n')
self.sleep(10)
code, out, err = cloudshell.run(self.username, self.password, 'createtable test_ingest\n')
self.assertEqual(code, 0)
Benchmark.setUp(self)
def start(command, stdin=None):
log.debug("Running %s", ' '.join(command))
handle = subprocess.Popen(command,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# remember the command for debugging
handle.command = command
return handle
def start(command, stdin=None):
log.debug("Running %s", ' '.join(command))
handle = subprocess.Popen(command,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# remember the command for debugging
handle.command = command
return handle
def setUp(self):
code, out, err = cloudshell.run(self.username, self.password, 'table test_ingest\n')
if out.find('does not exist') == -1:
log.debug("Deleting table test_ingest")
code, out, err = cloudshell.run(self.username, self.password, 'deletetable test_ingest -f\n')
self.assertEquals(code, 0, "Could not delete the table 'test_ingest'")
code, out, err = cloudshell.run(self.username, self.password, 'createtable test_ingest\n')
self.assertEqual(code, 0, "Could not create the table 'test_ingest'")
Benchmark.setUp(self)
def run(username, password, input):
"Run a command in accumulo"
handle = runner.start([path.accumulo('bin', 'accumulo'), 'shell -u %s -p %s' % (username, password) ],
stdin=subprocess.PIPE)
log.debug("Running: %r", input)
out, err = handle.communicate(input)
log.debug("Process finished: %d (%s)",
handle.returncode,
' '.join(handle.command))
return handle.returncode, out, err
def setUp(self):
for x in range(1, self.tables):
currentTable = 'test_ingest%d' % (x)
log.debug("Checking for table existence: %s" % currentTable)
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % currentTable)
if out.find('does not exist') == -1:
command = 'deletetable -f %s\n' % (currentTable)
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, 'Did not successfully delete table: %s' % currentTable)
Benchmark.setUp(self)
def run(username, password, input):
"Run a command in accumulo"
handle = runner.start([
path.accumulo('bin', 'accumulo'), 'shell', '-u', username, '-p',
password
],
stdin=subprocess.PIPE)
log.debug("Running: %r", input)
out, err = handle.communicate(input)
log.debug("Process finished: %d (%s)", handle.returncode,
' '.join(handle.command))
return handle.returncode, out, err
def runTest(self):
command = self.buildcommand(
'org.apache.accumulo.examples.simple.mapreduce.RowHash',
self.getInstance(), self.getZookeepers(), self.getUsername(),
self.getPassword(), self.input_table, 'column:columnqual',
self.output_table, self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode,
' '.join(handle.command))
return handle.returncode, out, err
def setUp(self):
code, out, err = cloudshell.run(self.username, self.password,
'table test_ingest\n')
if out.find('no such table') >= 0:
log.debug("Deleting table test_ingest")
code, out, err = cloudshell.run(self.username, self.password,
'deletetable test_ingest\n')
self.sleep(10)
code, out, err = cloudshell.run(self.username, self.password,
'createtable test_ingest\n')
self.assertEqual(code, 0)
Benchmark.setUp(self)
def setUp(self):
code, out, err = cloudshell.run(self.username, self.password,
'table test_ingest\n')
if out.find('does not exist') == -1:
log.debug("Deleting table test_ingest")
code, out, err = cloudshell.run(self.username, self.password,
'deletetable test_ingest -f\n')
self.assertEquals(code, 0,
"Could not delete the table 'test_ingest'")
code, out, err = cloudshell.run(self.username, self.password,
'createtable test_ingest\n')
self.assertEqual(code, 0, "Could not create the table 'test_ingest'")
Benchmark.setUp(self)
def runTest(self):
command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.RowHash',
self.getInstance(),
self.getZookeepers(),
self.getUsername(),
self.getPassword(),
self.input_table,
'column:columnqual',
self.output_table,
self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode, ' '.join(handle.command))
return handle.returncode, out, err
def setUp(self):
for x in range(1, self.tables):
currentTable = 'test_ingest%d' % (x)
log.debug("Checking for table existence: %s" % currentTable)
code, out, err = cloudshell.run(self.username, self.password,
'table %s\n' % currentTable)
if out.find('does not exist') == -1:
command = 'deletetable -f %s\n' % (currentTable)
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password,
command)
self.assertEqual(
code, 0,
'Did not successfully delete table: %s' % currentTable)
Benchmark.setUp(self)
def runTest(self):
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join( dir, 'splits' )
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.tablename, file))
command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
self.numrows(),
self.keysizemin(),
self.keysizemax(),
self.minvaluesize(),
self.maxvaluesize(),
self.tablename,
self.instance,
self.zookeepers,
self.username,
self.password,
self.numsplits)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode, ' '.join(handle.command))
return handle.returncode, out, err
def runTest(self):
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join(dir, 'splits')
code, out, err = cloudshell.run(
self.username, self.password,
"createtable %s -sf %s\n" % (self.tablename, file))
command = self.buildcommand(
'org.apache.accumulo.test.mapreduce.TeraSortIngest', '--count',
self.numrows(), '-nk', self.keysizemin(), '-xk',
self.keysizemax(), '-nv', self.minvaluesize(), '-xv',
self.maxvaluesize(), '-t', self.tablename, '-i', self.instance,
'-z', self.zookeepers, '-u', self.username, '-p', self.password,
'--splits', self.numsplits)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode,
' '.join(handle.command))
self.assertEqual(handle.returncode, 0,
"Job did not complete successfully")
return handle.returncode, out, err
def runTest(self):
commands = {}
for i, s in enumerate(slaveNames()):
commands[
s] = '%s %s -username %s -password %s -size %d -random %d %d %d %d' % (
accumulo('bin', 'accumulo'),
'org.apache.accumulo.server.test.TestIngest',
self.username, self.password, self.size(), self.random(),
self.count(), i * self.count(), 1)
results = runEach(commands)
codes = {}
for slave, (code, out, err) in results.items():
codes.setdefault(code, [])
codes[code].append(slave)
for code, slaves in codes.items():
if code != 0:
self.assertEqual(
code, 0,
"Bad exit code (%d) from slaves %r" % (code, slaves))
command = 'deletetable test_ingest\n'
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
def runTest(self):
commands = {}
for i, s in enumerate(slaveNames()):
commands[s] = '%s %s -username %s -password %s -size %d -random %d %d %d %d' % (
accumulo('bin', 'accumulo'),
'org.apache.accumulo.server.test.TestIngest',
self.username, self.password,
self.size(),
self.random(),
self.count(),
i*self.count(),
1)
results = runEach(commands)
codes = {}
for slave, (code, out, err) in results.items():
codes.setdefault(code, [])
codes[code].append(slave)
for code, slaves in codes.items():
if code != 0:
self.assertEqual(code, 0, "Bad exit code (%d) from slaves %r" % (code, slaves))
command = 'deletetable test_ingest\n'
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
def runTest(self):
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join(dir, "splits")
code, out, err = cloudshell.run(
self.username, self.password, "createtable %s -sf %s\n" % (self.tablename, file)
)
command = self.buildcommand(
"org.apache.accumulo.test.mapreduce.TeraSortIngest",
"--count",
self.numrows(),
"-nk",
self.keysizemin(),
"-xk",
self.keysizemax(),
"-nv",
self.minvaluesize(),
"-xv",
self.maxvaluesize(),
"-t",
self.tablename,
"-i",
self.instance,
"-z",
self.zookeepers,
"-u",
self.username,
"-p",
self.password,
"--splits",
self.numsplits,
)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode, " ".join(handle.command))
self.assertEqual(handle.returncode, 0, "Job did not complete successfully")
return handle.returncode, out, err
def runEach(commandMap):
result = {}
handles = []
for slave, command in commandMap.items():
log.debug("ssh: %s: %s", slave, command)
handle = subprocess.Popen(['ssh',slave] + [command],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for h in handle.stdout, handle.stderr:
fcntl.fcntl(h, fcntl.F_SETFL, os.O_NDELAY)
handle.slave = slave
handle.command = command
handle.start = time.time()
handles.append(handle)
handlesLeft = set(handles[:])
while handlesLeft:
fds = {}
doomed = set()
for h in handlesLeft:
more = []
if h.stdout != None:
more.append(h.stdout)
if h.stderr != None:
more.append(h.stderr)
for m in more:
fds[m] = h
if not more:
doomed.add(h)
handlesLeft -= doomed
if not handlesLeft: break
rd, wr, ex = select.select(fds.keys(), [], [], 10)
for r in rd:
handle = fds[r]
data = r.read(1024)
result.setdefault(handle, ['', ''])
if not data:
if r == handle.stdout:
handle.stdout = None
else:
handle.stderr = None
if r == handle.stdout:
result[handle][0] += data
else:
result[handle][1] += data
if handle.stdout == None and handle.stderr == None:
log.debug("Slave %s finished in %.2f",
handle.slave,
time.time() - handle.start)
handle.wait()
if not rd:
log.debug("Waiting on %d slaves (%s...)",
len(handlesLeft),
', '.join([h.slave for h in handlesLeft])[:50])
return dict([(h.slave, (h.returncode, out, err))
for h, (out, err) in result.items()])
def runEach(commandMap):
result = {}
handles = []
for slave, command in commandMap.items():
log.debug("ssh: %s: %s", slave, command)
handle = subprocess.Popen(['ssh',slave] + [command],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for h in handle.stdout, handle.stderr:
fcntl.fcntl(h, fcntl.F_SETFL, os.O_NDELAY)
handle.slave = slave
handle.command = command
handle.start = time.time()
handles.append(handle)
handlesLeft = set(handles[:])
while handlesLeft:
fds = {}
doomed = set()
for h in handlesLeft:
more = []
if h.stdout != None:
more.append(h.stdout)
if h.stderr != None:
more.append(h.stderr)
for m in more:
fds[m] = h
if not more:
doomed.add(h)
handlesLeft -= doomed
if not handlesLeft: break
rd, wr, ex = select.select(fds.keys(), [], [], 10)
for r in rd:
handle = fds[r]
data = r.read(1024)
result.setdefault(handle, ['', ''])
if not data:
if r == handle.stdout:
handle.stdout = None
else:
handle.stderr = None
if r == handle.stdout:
result[handle][0] += data
else:
result[handle][1] += data
if handle.stdout == None and handle.stderr == None:
log.debug("Slave %s finished in %.2f",
handle.slave,
time.time() - handle.start)
handle.wait()
if not rd:
log.debug("Waiting on %d slaves (%s...)",
len(handlesLeft),
', '.join([h.slave for h in handlesLeft])[:50])
return dict([(h.slave, (h.returncode, out, err))
for h, (out, err) in result.items()])
def runTest(self):
for x in range(1, self.tables):
currentTable = 'test_ingest%d' % (x)
command = 'createtable %s\n' % (currentTable)
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
# print err
for x in range(1, self.tables):
currentTable = 'test_ingest%d' % (x)
command = 'deletetable %s\n' % (currentTable)
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
# print err
log.debug("Process finished")
return code, out, err
def tearDown(self):
command = 'deletetable test_ingest -f\n'
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, "Could not delete the table 'test_ingest'")
Benchmark.tearDown(self)
def setUp(self):
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.tablename)
if out.find('does not exist') == -1:
log.debug('Deleting table %s' % self.tablename)
code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' % self.tablename)
Benchmark.setUp(self)
def runAll(command):
slaves = slaveNames()
log.debug("Running %s on %s..", command, ', '.join(slaves)[:50])
return runEach(dict([(s, command) for s in slaves]))
def runAll(command):
tservers = tserverNames()
log.debug("Running %s on %s..", command, ', '.join(tservers)[:50])
return runEach(dict([(s, command) for s in tservers]))
def runTest(self):
command = 'createtable %s -sf %s\n' % (self.tablename, self.splitsfile)
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, 'Could not create table: %s' % out)
return code, out, err
def runAll(command):
tservers = tserverNames()
log.debug("Running %s on %s..", command, ', '.join(tservers)[:50])
return runEach(dict([(s, command) for s in tservers]))
def runAll(command):
slaves = slaveNames()
log.debug("Running %s on %s..", command, ', '.join(slaves)[:50])
return runEach(dict([(s, command) for s in slaves]))
def runTest(self):
command = 'createtable %s -sf %s\n' % (self.tablename, self.splitsfile)
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, 'Could not create table: %s' % out)
return code, out, err
def tearDown(self):
command = 'deletetable test_ingest -f\n'
log.debug("Running Command %r", command)
code, out, err = cloudshell.run(self.username, self.password, command)
self.assertEqual(code, 0, "Could not delete the table 'test_ingest'")
Benchmark.tearDown(self)
def setUp(self):
code, out, err = cloudshell.run(self.username, self.password, "table %s\n" % self.tablename)
if out.find("does not exist") == -1:
log.debug("Deleting table %s" % self.tablename)
code, out, err = cloudshell.run(self.username, self.password, "deletetable -f %s\n" % self.tablename)
Benchmark.setUp(self)