Beispiel #1
0
 def setUp(self): 
     dir = os.path.dirname(os.path.realpath(__file__))
     file = os.path.join( dir, 'splits' )  
     code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.input_table) 
     if out.find('does not exist') == -1:
        code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' % self.input_table) 
        self.sleep(15)
     code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.input_table, file))
     code, out, err = cloudshell.run(self.username, self.password, 'table %s\n' % self.output_table) 
     if out.find('does not exist') == -1:
         code, out, err = cloudshell.run(self.username, self.password, 'deletetable -f %s\n' %
                 self.output_table) 
         self.sleep(15)
     code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.output_table, file))
     command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
                                 '--count', self.numrows(),
                                 '-nk', self.keysizemin(),
                                 '-xk', self.keysizemax(),
                                 '-nv', self.minvaluesize(),
                                 '-xv', self.maxvaluesize(),
                                 '--table', self.input_table, 
                                 '-i', self.getInstance(),
                                 '-z', self.getZookeepers(),
                                 '-u', self.getUsername(),
                                 '-p', self.getPassword(),
                                 '--splits', self.maxmaps)
     handle = runner.start(command, stdin=subprocess.PIPE)
     log.debug("Running: %r", command)
     out, err = handle.communicate("")  
     Benchmark.setUp(self)
Beispiel #2
0
 def setUp(self): 
     random.jumpahead(int(time.time()))
     num = random.randint(1, 100000)
     self.input_table = self.input_table + "_" + str(num) 
     self.output_table = self.output_table + "_" + str(num)    
     #if (not os.getenv("HADOOP_CLASSPATH")):
     #    os.putenv("HADOOP_CLASSPATH", self.getjars(":"))
     dir = os.path.dirname(os.path.realpath(__file__))
     file = os.path.join( dir, 'splits' )  
     # code, out, err = cloudshell.run(self.username, self.password, 'table RowHashTestInput\n') 
     # if out.find('no such table') == -1:
     #    code, out, err = cloudshell.run(self.username, self.password, 'deletetable RowHashTestInput\n') 
     #    self.sleep(15)
     code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.input_table, file))
     #code, out, err = cloudshell.run('table RowHashTest\n') 
     #if out.find('no such table') == -1:
     #    code, out, err = cloudshell.run('user root\nsecret\ndeletetable RowHashTest\n') 
     #    self.sleep(15)
     code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.output_table, file))
     command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
                                 self.numrows(),
                                 self.keysizemin(),
                                 self.keysizemax(),
                                 self.minvaluesize(),
                                 self.maxvaluesize(),
                                 self.input_table, 
                                 self.getInstance(),
                                 self.getZookeepers(),
                                 self.getUsername(),
                                 self.getPassword(),
                                 self.maxmaps)
     handle = runner.start(command, stdin=subprocess.PIPE)
     log.debug("Running: %r", command)
     out, err = handle.communicate("")  
     Benchmark.setUp(self)
Beispiel #3
0
 def setUp(self):
     random.jumpahead(int(time.time()))
     num = random.randint(1, 100000)
     self.input_table = self.input_table + "_" + str(num)
     self.output_table = self.output_table + "_" + str(num)
     #if (not os.getenv("HADOOP_CLASSPATH")):
     #    os.putenv("HADOOP_CLASSPATH", self.getjars(":"))
     dir = os.path.dirname(os.path.realpath(__file__))
     file = os.path.join(dir, 'splits')
     # code, out, err = cloudshell.run(self.username, self.password, 'table RowHashTestInput\n')
     # if out.find('no such table') == -1:
     #    code, out, err = cloudshell.run(self.username, self.password, 'deletetable RowHashTestInput\n')
     #    self.sleep(15)
     code, out, err = cloudshell.run(
         self.username, self.password,
         "createtable %s -sf %s\n" % (self.input_table, file))
     #code, out, err = cloudshell.run('table RowHashTest\n')
     #if out.find('no such table') == -1:
     #    code, out, err = cloudshell.run('user root\nsecret\ndeletetable RowHashTest\n')
     #    self.sleep(15)
     code, out, err = cloudshell.run(
         self.username, self.password,
         "createtable %s -sf %s\n" % (self.output_table, file))
     command = self.buildcommand(
         'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
         self.numrows(), self.keysizemin(), self.keysizemax(),
         self.minvaluesize(), self.maxvaluesize(), self.input_table,
         self.getInstance(), self.getZookeepers(), self.getUsername(),
         self.getPassword(), self.maxmaps)
     handle = runner.start(command, stdin=subprocess.PIPE)
     log.debug("Running: %r", command)
     out, err = handle.communicate("")
     Benchmark.setUp(self)
Beispiel #4
0
def run(username, password, input):
    "Run a command in accumulo"
    handle = runner.start([path.accumulo('bin', 'accumulo'), 'shell -u %s -p %s' % (username, password) ],
                          stdin=subprocess.PIPE)
    log.debug("Running: %r", input)
    out, err = handle.communicate(input)
    log.debug("Process finished: %d (%s)",
              handle.returncode,
              ' '.join(handle.command))
    return handle.returncode, out, err
 def runTest(self):
     command = self.buildcommand(
         'org.apache.accumulo.examples.simple.mapreduce.RowHash',
         self.getInstance(), self.getZookeepers(), self.getUsername(),
         self.getPassword(), self.input_table, 'column:columnqual',
         self.output_table, self.maxmaps)
     handle = runner.start(command, stdin=subprocess.PIPE)
     log.debug("Running: %r", command)
     out, err = handle.communicate("")
     log.debug("Process finished: %d (%s)", handle.returncode,
               ' '.join(handle.command))
     return handle.returncode, out, err
Beispiel #6
0
def run(username, password, input):
    "Run a command in accumulo"
    handle = runner.start([
        path.accumulo('bin', 'accumulo'), 'shell', '-u', username, '-p',
        password
    ],
                          stdin=subprocess.PIPE)
    log.debug("Running: %r", input)
    out, err = handle.communicate(input)
    log.debug("Process finished: %d (%s)", handle.returncode,
              ' '.join(handle.command))
    return handle.returncode, out, err
 def runTest(self):   
     command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.RowHash',
                                 self.getInstance(),
                                 self.getZookeepers(),
                                 self.getUsername(),
                                 self.getPassword(),
                                 self.input_table,
                                 'column:columnqual',
                                 self.output_table,
                                 self.maxmaps)
     handle = runner.start(command, stdin=subprocess.PIPE)        
     log.debug("Running: %r", command)
     out, err = handle.communicate("")
     log.debug("Process finished: %d (%s)", handle.returncode, ' '.join(handle.command))
     return handle.returncode, out, err
 def runTest(self):        
     dir = os.path.dirname(os.path.realpath(__file__))
     file = os.path.join( dir, 'splits' )
     code, out, err = cloudshell.run(self.username, self.password, "createtable %s -sf %s\n" % (self.tablename, file))
     command = self.buildcommand('org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
                                 self.numrows(),
                                 self.keysizemin(),
                                 self.keysizemax(),
                                 self.minvaluesize(),
                                 self.maxvaluesize(),
                                 self.tablename,
                                 self.instance,
                                 self.zookeepers,
                                 self.username,
                                 self.password, 
                                 self.numsplits)
     handle = runner.start(command, stdin=subprocess.PIPE)
     log.debug("Running: %r", command)
     out, err = handle.communicate("")
     log.debug("Process finished: %d (%s)", handle.returncode, ' '.join(handle.command))
     return handle.returncode, out, err
 def runTest(self):
     dir = os.path.dirname(os.path.realpath(__file__))
     file = os.path.join(dir, 'splits')
     code, out, err = cloudshell.run(
         self.username, self.password,
         "createtable %s -sf %s\n" % (self.tablename, file))
     command = self.buildcommand(
         'org.apache.accumulo.test.mapreduce.TeraSortIngest', '--count',
         self.numrows(), '-nk', self.keysizemin(), '-xk',
         self.keysizemax(), '-nv', self.minvaluesize(), '-xv',
         self.maxvaluesize(), '-t', self.tablename, '-i', self.instance,
         '-z', self.zookeepers, '-u', self.username, '-p', self.password,
         '--splits', self.numsplits)
     handle = runner.start(command, stdin=subprocess.PIPE)
     log.debug("Running: %r", command)
     out, err = handle.communicate("")
     log.debug("Process finished: %d (%s)", handle.returncode,
               ' '.join(handle.command))
     self.assertEqual(handle.returncode, 0,
                      "Job did not complete successfully")
     return handle.returncode, out, err
Beispiel #10
0
 def runTest(self):
     dir = os.path.dirname(os.path.realpath(__file__))
     file = os.path.join(dir, "splits")
     code, out, err = cloudshell.run(
         self.username, self.password, "createtable %s -sf %s\n" % (self.tablename, file)
     )
     command = self.buildcommand(
         "org.apache.accumulo.test.mapreduce.TeraSortIngest",
         "--count",
         self.numrows(),
         "-nk",
         self.keysizemin(),
         "-xk",
         self.keysizemax(),
         "-nv",
         self.minvaluesize(),
         "-xv",
         self.maxvaluesize(),
         "-t",
         self.tablename,
         "-i",
         self.instance,
         "-z",
         self.zookeepers,
         "-u",
         self.username,
         "-p",
         self.password,
         "--splits",
         self.numsplits,
     )
     handle = runner.start(command, stdin=subprocess.PIPE)
     log.debug("Running: %r", command)
     out, err = handle.communicate("")
     log.debug("Process finished: %d (%s)", handle.returncode, " ".join(handle.command))
     self.assertEqual(handle.returncode, 0, "Job did not complete successfully")
     return handle.returncode, out, err