def setUp(self):
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join( dir, 'splits' )
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.input_table)
if out.find('does not exist') == -1:
code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' % self.input_table)
self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.input_table, file))
code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.output_table)
if out.find('does not exist') == -1:
code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' %
self.output_table)
self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.output_table, file))
command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
'--count', self.numrows(),
'-nk', self.keysizemin(),
'-xk', self.keysizemax(),
'-nv', self.minvaluesize(),
'-xv', self.maxvaluesize(),
'--table', self.input_table,
'-i', self.getInstance(),
'-z', self.getZookeepers(),
'-u', self.getUsername(),
'-p', self.getPassword(),
'--splits', self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
Benchmark.setUp(self)
def setUp(self):
random.jumpahead(int(time.time()))
num = random.randint(1, 100000)
self.input_table = self.input_table + "_" + str(num)
self.output_table = self.output_table + "_" + str(num)
#if (not os.getenv("HADOOP_CLASSPATH")):
# os.putenv("HADOOP_CLASSPATH", self.getjars(":"))
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join( dir, 'splits' )
# code, out, err = cloudshell.run(self.username, self.password, 'table RowHashTestInput\n')
# if out.find('no such table') == -1:
# code, out, err = cloudshell.run(self.username, self.password, 'deletetable RowHashTestInput\n')
# self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.input_table, file))
#code, out, err = cloudshell.run('table RowHashTest\n')
#if out.find('no such table') == -1:
# code, out, err = cloudshell.run('user root\nsecret\ndeletetable RowHashTest\n')
# self.sleep(15)
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.output_table, file))
command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
self.numrows(),
self.keysizemin(),
self.keysizemax(),
self.minvaluesize(),
self.maxvaluesize(),
self.input_table,
self.getInstance(),
self.getZookeepers(),
self.getUsername(),
self.getPassword(),
self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
Benchmark.setUp(self)
def setUp(self):
random.jumpahead(int(time.time()))
num = random.randint(1, 100000)
self.input_table = self.input_table + "_" + str(num)
self.output_table = self.output_table + "_" + str(num)
#if (not os.getenv("HADOOP_CLASSPATH")):
# os.putenv("HADOOP_CLASSPATH", self.getjars(":"))
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join(dir, 'splits')
# code, out, err = cloudshell.run(self.username, self.password, 'table RowHashTestInput\n')
# if out.find('no such table') == -1:
# code, out, err = cloudshell.run(self.username, self.password, 'deletetable RowHashTestInput\n')
# self.sleep(15)
code, out, err = cloudshell.run(
self.username, self.password,
"createtable %s -sf %s\n" % (self.input_table, file))
#code, out, err = cloudshell.run('table RowHashTest\n')
#if out.find('no such table') == -1:
# code, out, err = cloudshell.run('user root\nsecret\ndeletetable RowHashTest\n')
# self.sleep(15)
code, out, err = cloudshell.run(
self.username, self.password,
"createtable %s -sf %s\n" % (self.output_table, file))
command = self.buildcommand(
'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
self.numrows(), self.keysizemin(), self.keysizemax(),
self.minvaluesize(), self.maxvaluesize(), self.input_table,
self.getInstance(), self.getZookeepers(), self.getUsername(),
self.getPassword(), self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
Benchmark.setUp(self)
def run(username, password, input):
"Run a command in accumulo"
handle = runner.start([path.accumulo('bin', 'accumulo'), 'shell -u %s -p %s' % (username, password) ],
stdin=subprocess.PIPE)
log.debug("Running: %r", input)
out, err = handle.communicate(input)
log.debug("Process finished: %d (%s)",
handle.returncode,
' '.join(handle.command))
return handle.returncode, out, err
def runTest(self):
command = self.buildcommand(
'org.apache.accumulo.examples.simple.mapreduce.RowHash',
self.getInstance(), self.getZookeepers(), self.getUsername(),
self.getPassword(), self.input_table, 'column:columnqual',
self.output_table, self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode,
' '.join(handle.command))
return handle.returncode, out, err
def run(username, password, input):
"Run a command in accumulo"
handle = runner.start([
path.accumulo('bin', 'accumulo'), 'shell', '-u', username, '-p',
password
],
stdin=subprocess.PIPE)
log.debug("Running: %r", input)
out, err = handle.communicate(input)
log.debug("Process finished: %d (%s)", handle.returncode,
' '.join(handle.command))
return handle.returncode, out, err
def runTest(self):
command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.RowHash',
self.getInstance(),
self.getZookeepers(),
self.getUsername(),
self.getPassword(),
self.input_table,
'column:columnqual',
self.output_table,
self.maxmaps)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode, ' '.join(handle.command))
return handle.returncode, out, err
def runTest(self):
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join( dir, 'splits' )
code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.tablename, file))
command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
self.numrows(),
self.keysizemin(),
self.keysizemax(),
self.minvaluesize(),
self.maxvaluesize(),
self.tablename,
self.instance,
self.zookeepers,
self.username,
self.password,
self.numsplits)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode, ' '.join(handle.command))
return handle.returncode, out, err
def runTest(self):
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join(dir, 'splits')
code, out, err = cloudshell.run(
self.username, self.password,
"createtable %s -sf %s\n" % (self.tablename, file))
command = self.buildcommand(
'org.apache.accumulo.test.mapreduce.TeraSortIngest', '--count',
self.numrows(), '-nk', self.keysizemin(), '-xk',
self.keysizemax(), '-nv', self.minvaluesize(), '-xv',
self.maxvaluesize(), '-t', self.tablename, '-i', self.instance,
'-z', self.zookeepers, '-u', self.username, '-p', self.password,
'--splits', self.numsplits)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode,
' '.join(handle.command))
self.assertEqual(handle.returncode, 0,
"Job did not complete successfully")
return handle.returncode, out, err
def runTest(self):
dir = os.path.dirname(os.path.realpath(__file__))
file = os.path.join(dir, "splits")
code, out, err = cloudshell.run(
self.username, self.password, "createtable %s -sf %s\n" % (self.tablename, file)
)
command = self.buildcommand(
"org.apache.accumulo.test.mapreduce.TeraSortIngest",
"--count",
self.numrows(),
"-nk",
self.keysizemin(),
"-xk",
self.keysizemax(),
"-nv",
self.minvaluesize(),
"-xv",
self.maxvaluesize(),
"-t",
self.tablename,
"-i",
self.instance,
"-z",
self.zookeepers,
"-u",
self.username,
"-p",
self.password,
"--splits",
self.numsplits,
)
handle = runner.start(command, stdin=subprocess.PIPE)
log.debug("Running: %r", command)
out, err = handle.communicate("")
log.debug("Process finished: %d (%s)", handle.returncode, " ".join(handle.command))
self.assertEqual(handle.returncode, 0, "Job did not complete successfully")
return handle.returncode, out, err