예제 #1
0
파일: cmd.py 프로젝트: CyaLiven/dumbo
def start(prog,
          opts,
          stdout=sys.stdout,
          stderr=sys.stderr):

    opts = Options(opts)
    opts += Options(configopts('common'))
    opts += Options(configopts('start'))

    pyenv = envdef('PYTHONPATH', opts['libegg'],
                   shortcuts=dict(configopts('eggs', prog)),
                   extrapaths=sys.path)

    if not opts['prog']:
        opts.add('prog', prog)

    if not os.path.exists(prog):
        if prog.endswith(".py"):
            print >> sys.stderr, 'ERROR:', prog, 'does not exist'
            return 1
        prog = '-m ' + prog

    return execute("%s %s" % (sys.executable, prog),
                   opts,
                   pyenv,
                   stdout=stdout,
                   stderr=stderr,
                   printcmd=False)
예제 #2
0
파일: cmd.py 프로젝트: soxofaan/dumbo
def start(prog, opts, stdout=sys.stdout, stderr=sys.stderr):

    opts = Options(opts)
    opts += Options(configopts('common'))
    opts += Options(configopts('start'))

    pyenv = envdef('PYTHONPATH',
                   opts['libegg'],
                   shortcuts=dict(configopts('eggs', prog)),
                   extrapaths=sys.path)

    if not opts['prog']:
        opts.add('prog', prog)

    if not os.path.exists(prog):
        if prog.endswith(".py"):
            print >> sys.stderr, 'ERROR:', prog, 'does not exist'
            return 1
        prog = '-m ' + prog

    return execute("%s %s" % (sys.executable, prog),
                   opts,
                   pyenv,
                   stdout=stdout,
                   stderr=stderr,
                   printcmd=False)
예제 #3
0
파일: streaming.py 프로젝트: dgleich/dumbo
 def create_filesystem(self, opts):
     hadoopopt = getopt(opts, 'hadoop', delete=False)
     hadoopshort = hadoopopt[0]
     hadoopdir = findhadoop(hadoopopt[0])
     allopts = configopts('streaming')
     allopts += configopts('streaming_' + hadoopshort)
     streamingjar = getopt(allopts, 'streamingjar')
     if streamingjar:
         streamingjar = streamingjar[0]
     return StreamingFileSystem(hadoopdir, streamingjar)
예제 #4
0
파일: streaming.py 프로젝트: mlerner/dumbo
    def cat(self, path, opts):
        streamingjar = findjar(
            self.hadoop, 'streaming',
            opts['hadooplib'] if 'hadooplib' in opts else None)
        if not streamingjar:
            print >> sys.stderr, 'ERROR: Streaming jar not found'
            return 1
        hadenv = envdef('HADOOP_CLASSPATH',
                        opts['libjar'],
                        shortcuts=dict(configopts('jars')))
        try:
            import typedbytes
            ls = os.popen('%s %s -ls %s' % (hadenv, self.hdfs, path))
            subpaths = [line.split()[-1] for line in ls if ":" in line]
            ls.close()
            for subpath in subpaths:
                if subpath.split("/")[-1].startswith("_"):
                    continue
                dumptb = os.popen(
                    '%s %s/bin/hadoop jar %s dumptb %s 2> /dev/null' %
                    (hadenv, self.hadoop, streamingjar, subpath))

                dump = dumpcode if 'yes' in opts['ascode'] else dumptext
                outputs = dump(typedbytes.PairedInput(dumptb))

                for output in outputs:
                    print '\t'.join(output)
                dumptb.close()
        except IOError:
            pass  # ignore
        return 0
예제 #5
0
파일: streaming.py 프로젝트: CyaLiven/dumbo
    def cat(self, path, opts):
        streamingjar = findjar(self.hadoop, 'streaming',
                               opts['hadooplib'] if 'hadooplib' in opts else None)
        if not streamingjar:
            print >> sys.stderr, 'ERROR: Streaming jar not found'
            return 1
        hadenv = envdef('HADOOP_CLASSPATH', opts['libjar'],
            shortcuts=dict(configopts('jars')))
        try:
            import typedbytes

            if "combined" in opts and opts["combined"][0] == "yes":
                subpaths = [path]
            else:
                ls = os.popen('%s %s -ls %s' % (hadenv, self.hdfs, path))
                subpaths = [line.split()[-1] for line in ls if ":" in line]
                ls.close()

            for subpath in subpaths:
                if subpath.split("/")[-1].startswith("_"):
                    continue
                dumptb = os.popen('%s %s/bin/hadoop jar %s dumptb %s 2> /dev/null'
                                  % (hadenv, self.hadoop, streamingjar, subpath))

                dump = dumpcode if 'yes' in opts['ascode'] else dumptext
                outputs = dump(typedbytes.PairedInput(dumptb))

                for output in outputs:
                    print '\t'.join(output)
                dumptb.close()
        except IOError:
            pass  # ignore
        return 0
예제 #6
0
 def cat(self, path, opts):
     addedopts = getopts(opts, ['libjar'], delete=False)
     streamingjar = findjar(self.hadoop, 'streaming')
     if not streamingjar:
         print >> sys.stderr, 'ERROR: Streaming jar not found'
         return 1
     hadenv = envdef('HADOOP_CLASSPATH', addedopts['libjar'],
                     shortcuts=dict(configopts('jars')))
     try:
         import typedbytes
         ls = os.popen('%s %s/bin/hadoop dfs -ls %s' % (hadenv, self.hadoop, path))
         if sum(c in path for c in ("*", "?", "{")) > 0:
             # cat each file separately when the path contains special chars
             lineparts = (line.split()[-1] for line in ls)
             subpaths = [part for part in lineparts if part.startswith("/")]
         else:
             # we still do the ls even in this case to make sure we print errors 
             subpaths = [path]
         ls.close()
         for subpath in subpaths:
             dumptb = os.popen('%s %s/bin/hadoop jar %s dumptb %s 2> /dev/null'
                               % (hadenv, self.hadoop, streamingjar, subpath))
             ascodeopt = getopt(opts, 'ascode')
             if ascodeopt and ascodeopt[0] == 'yes':
                 outputs = dumpcode(typedbytes.PairedInput(dumptb))
             else:
                 outputs = dumptext(typedbytes.PairedInput(dumptb))
             for output in outputs:
                 print '\t'.join(output)
             dumptb.close()
     except IOError:
         pass  # ignore
     return 0
예제 #7
0
파일: streaming.py 프로젝트: ebottabi/dumbo
    def cat(self, path, opts):
        streamingjar = findjar(self.hadoop, 'streaming')
        if not streamingjar:
            print >> sys.stderr, 'ERROR: Streaming jar not found'
            return 1
        hadenv = envdef('HADOOP_CLASSPATH',
                        opts['libjar'],
                        shortcuts=dict(configopts('jars')))
        try:
            import typedbytes
            ls = os.popen('%s %s dfs -ls %s' % (hadenv, self.hdfs, path))
            if sum(c in path for c in ("*", "?", "{")) > 0:
                # cat each file separately when the path contains special chars
                lineparts = (line.split()[-1] for line in ls)
                subpaths = [part for part in lineparts if part.startswith("/")]
            else:
                # we still do the ls even in this case to make sure we print errors
                subpaths = [path]
            ls.close()
            for subpath in subpaths:
                if subpath.endswith("/_logs"):
                    continue
                dumptb = os.popen(
                    '%s %s/bin/hadoop jar %s dumptb %s 2> /dev/null' %
                    (hadenv, self.hadoop, streamingjar, subpath))

                dump = dumpcode if 'yes' in opts['ascode'] else dumptext
                outputs = dump(typedbytes.PairedInput(dumptb))

                for output in outputs:
                    print '\t'.join(output)
                dumptb.close()
        except IOError:
            pass  # ignore
        return 0
    def run(self):
        retval = Iteration.run(self)
        if retval != 0:
            return retval
        addedopts = getopts(self.opts, [
            'input', 'output', 'mapper', 'reducer', 'libegg', 'delinputs',
            'cmdenv', 'pv', 'addpath', 'inputformat', 'outputformat',
            'numreducetasks', 'python', 'pypath', 'sorttmpdir', 'sortbufsize'
        ])
        (mapper, reducer) = (addedopts['mapper'][0], addedopts['reducer'][0])
        if not addedopts['input'] or not addedopts['output']:
            print >> sys.stderr, 'ERROR: input or output not specified'
            return 1
        inputs = reduce(operator.concat,
                        (input.split(' ') for input in addedopts['input']))
        output = addedopts['output'][0]
        pyenv = envdef('PYTHONPATH',
                       addedopts['libegg'],
                       shortcuts=dict(configopts('eggs', self.prog)),
                       extrapaths=addedopts['pypath'])
        cmdenv = ' '.join("%s='%s'" % tuple(arg.split('='))
                          for arg in addedopts['cmdenv'])
        if addedopts['pv'] and addedopts['pv'][0] == 'yes':
            mpv = '| pv -s `du -b %s | cut -f 1` -cN map ' % ' '.join(inputs)
            (spv, rpv) = ('| pv -cN sort ', '| pv -cN reduce ')
        else:
            (mpv, spv, rpv) = ('', '', '')

        (sorttmpdir, sortbufsize) = ('', '')
        if addedopts['sorttmpdir']:
            sorttmpdir = "-T %s" % addedopts['sorttmpdir'][0]
        if addedopts['sortbufsize']:
            sortbufsize = "-S %s" % addedopts['sortbufsize'][0]

        python = addedopts['python'][0]
        encodepipe = pyenv + ' ' + python + \
                     ' -m dumbo.cmd encodepipe -file ' + ' -file '.join(inputs)
        if addedopts['inputformat'] and addedopts['inputformat'][0] == 'code':
            encodepipe += ' -alreadycoded yes'
        if addedopts['addpath'] and addedopts['addpath'][0] != 'no':
            encodepipe += ' -addpath yes'
        if addedopts['numreducetasks'] and addedopts['numreducetasks'][
                0] == '0':
            retval = execute("%s | %s %s %s %s > '%s'" %
                             (encodepipe, pyenv, cmdenv, mapper, mpv, output))
        else:
            retval = execute(
                "%s | %s %s %s %s| LC_ALL=C sort %s %s %s| %s %s %s %s> '%s'" %
                (encodepipe, pyenv, cmdenv, mapper, mpv, sorttmpdir,
                 sortbufsize, spv, pyenv, cmdenv, reducer, rpv, output))
        if addedopts['delinputs'] and addedopts['delinputs'][0] == 'yes':
            for file in addedopts['input']:
                execute('rm ' + file)
        return retval
예제 #9
0
파일: common.py 프로젝트: nbyloff/dumbo
    def run(self):
        addedopts = getopts(
            self.opts,
            [
                "fake",
                "debug",
                "python",
                "iteration",
                "itercount",
                "hadoop",
                "starter",
                "name",
                "memlimit",
                "param",
                "parser",
                "record",
                "joinkeys",
                "hadoopconf",
                "mapper",
                "reducer",
            ],
        )
        if addedopts["fake"] and addedopts["fake"][0] == "yes":

            def dummysystem(*args, **kwargs):
                return 0

            global system
            system = dummysystem  # not very clean, but it works...
        if addedopts["debug"] and addedopts["debug"][0] == "yes":
            self.opts.append(("cmdenv", "dumbo_debug=yes"))
        if not addedopts["python"]:
            python = "python"
        else:
            python = addedopts["python"][0]
        self.opts.append(("python", python))
        if not addedopts["iteration"]:
            iter = 0
        else:
            iter = int(addedopts["iteration"][0])
        if not addedopts["itercount"]:
            itercnt = 1
        else:
            itercnt = int(addedopts["itercount"][0])
        if addedopts["name"]:
            name = addedopts["name"][0]
        else:
            name = self.prog.split("/")[-1]
        self.opts.append(("name", "%s (%s/%s)" % (name, iter + 1, itercnt)))
        if not addedopts["hadoop"]:
            pypath = "/".join(self.prog.split("/")[:-1])
            if pypath:
                self.opts.append(("pypath", pypath))
        else:
            self.opts.append(("hadoop", addedopts["hadoop"][0]))
        progmod = self.prog.split("/")[-1]
        progmod = progmod[:-3] if progmod.endswith(".py") else progmod
        memlim = " 262144000"  # 250MB limit by default
        if addedopts["memlimit"]:
            # Limit amount of memory. This supports syntax
            # of the form '256m', '12g' etc.
            try:
                _memlim = int(addedopts["memlimit"][0][:-1])
                memlim = (
                    " %i"
                    % {"g": 1073741824 * _memlim, "m": 1048576 * _memlim, "k": 1024 * _memlim, "b": 1 * _memlim}[
                        addedopts["memlimit"][0][-1].lower()
                    ]
                )
            except KeyError:
                # Assume specified in bytes by default
                memlim = " " + addedopts["memlimit"][0]

        if addedopts["mapper"]:
            self.opts.append(("mapper", addedopts["mapper"][0]))
        else:
            self.opts.append(("mapper", "%s -m %s map %i%s" % (python, progmod, iter, memlim)))
        if addedopts["reducer"]:
            self.opts.append(("reducer", addedopts["reducer"][0]))
        else:
            self.opts.append(("reducer", "%s -m %s red %i%s" % (python, progmod, iter, memlim)))
        for param in addedopts["param"]:
            self.opts.append(("cmdenv", param))
        if addedopts["parser"] and iter == 0:
            parser = addedopts["parser"][0]
            shortcuts = dict(configopts("parsers", self.prog))
            if parser in shortcuts:
                parser = shortcuts[parser]
            self.opts.append(("cmdenv", "dumbo_parser=" + parser))
        if addedopts["record"] and iter == 0:
            record = addedopts["record"][0]
            shortcuts = dict(configopts("records", self.prog))
            if record in shortcuts:
                record = shortcuts[record]
            self.opts.append(("cmdenv", "dumbo_record=" + record))
        if addedopts["joinkeys"] and addedopts["joinkeys"][0] == "yes":
            self.opts.append(("cmdenv", "dumbo_joinkeys=yes"))
            self.opts.append(("partitioner", "org.apache.hadoop.mapred.lib.BinaryPartitioner"))
            self.opts.append(("jobconf", "mapred.binary.partitioner.right.offset=-6"))
        for hadoopconf in addedopts["hadoopconf"]:
            self.opts.append(("jobconf", hadoopconf))
        self.opts.append(("libegg", re.sub("\.egg.*$", ".egg", __file__)))
        return 0
예제 #10
0
 def run(self):
     retval = Iteration.run(self)
     if retval != 0:
         return retval
     if os.path.exists(self.prog):
         self.opts.append(('file', self.prog))
     addedopts = getopts(self.opts, ['hadoop',
                                     'name',
                                     'delinputs',
                                     'libegg',
                                     'libjar',
                                     'inputformat',
                                     'outputformat',
                                     'nummaptasks',
                                     'numreducetasks',
                                     'priority',
                                     'queue',
                                     'cachefile',
                                     'cachearchive',
                                     'file',
                                     'codewritable',
                                     'addpath',
                                     'getpath',
                                     'python',
                                     'streamoutput',
                                     'pypath'])
     hadoop = findhadoop(addedopts['hadoop'][0])
     streamingjar = findjar(hadoop, 'streaming')
     if not streamingjar:
         print >> sys.stderr, 'ERROR: Streaming jar not found'
         return 1
     try: 
         import typedbytes
     except ImportError:
         print >> sys.stderr, 'ERROR: "typedbytes" module not found'
         return 1
     modpath = re.sub('\.egg.*$', '.egg', typedbytes.__file__)
     if modpath.endswith('.egg'):            
         addedopts['libegg'].append(modpath)    
     else:
         self.opts.append(('file', modpath)) 
     self.opts.append(('jobconf', 'stream.map.input=typedbytes'))
     self.opts.append(('jobconf', 'stream.reduce.input=typedbytes'))
     if addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0':
         self.opts.append(('jobconf', 'stream.reduce.output=typedbytes'))
         if addedopts['streamoutput']:
             id_ = addedopts['streamoutput'][0]
             self.opts.append(('jobconf', 'stream.map.output=' + id_))
         else: 
             self.opts.append(('jobconf', 'stream.map.output=typedbytes'))
     else:
         self.opts.append(('jobconf', 'stream.map.output=typedbytes'))
         if addedopts['streamoutput']:
             id_ = addedopts['streamoutput'][0]
             self.opts.append(('jobconf', 'stream.reduce.output=' + id_))
         else:
             self.opts.append(('jobconf', 'stream.reduce.output=typedbytes'))
     if not addedopts['name']:
         self.opts.append(('jobconf', 'mapred.job.name='
                           + self.prog.split('/')[-1]))
     else:
         self.opts.append(('jobconf', 'mapred.job.name=%s'
                           % addedopts['name'][0]))
     if addedopts['nummaptasks']:
         self.opts.append(('jobconf', 'mapred.map.tasks=%s'
                           % addedopts['nummaptasks'][0]))
     if addedopts['numreducetasks']:
         numreducetasks = int(addedopts['numreducetasks'][0])
         self.opts.append(('numReduceTasks', str(numreducetasks)))
     if addedopts['priority']:
         self.opts.append(('jobconf', 'mapred.job.priority=%s'
                           % addedopts['priority'][0]))
     if addedopts['queue']:
         self.opts.append(('jobconf', 'mapred.job.queue.name=%s'
                           % addedopts['queue'][0]))
     if addedopts['cachefile']:
         for cachefile in addedopts['cachefile']:
             self.opts.append(('cacheFile', cachefile))
     if addedopts['cachearchive']:
         for cachearchive in addedopts['cachearchive']:
             self.opts.append(('cacheArchive', cachearchive))
     if addedopts['file']:
         for file in addedopts['file']:
             if not '://' in file:
                 if not os.path.exists(file):
                     raise ValueError('file "' + file + '" does not exist')
                 file = 'file://' + os.path.abspath(file)
             self.opts.append(('file', file))
     if not addedopts['inputformat']:
         addedopts['inputformat'] = ['auto']
     inputformat_shortcuts = \
         {'code': 'org.apache.hadoop.streaming.AutoInputFormat',
          'text': 'org.apache.hadoop.mapred.TextInputFormat',
          'sequencefile': 'org.apache.hadoop.streaming.AutoInputFormat',
          'auto': 'org.apache.hadoop.streaming.AutoInputFormat'}
     inputformat_shortcuts.update(configopts('inputformats', self.prog))
     inputformat = addedopts['inputformat'][0]
     if inputformat_shortcuts.has_key(inputformat.lower()):
         inputformat = inputformat_shortcuts[inputformat.lower()]
     self.opts.append(('inputformat', inputformat))
     if not addedopts['outputformat']:
         addedopts['outputformat'] = ['sequencefile']
     if addedopts['getpath'] and addedopts['getpath'] != 'no':
         outputformat_shortcuts = \
             {'code': 'fm.last.feathers.output.MultipleSequenceFiles',
              'text': 'fm.last.feathers.output.MultipleTextFiles',               
              'raw': 'fm.last.feathers.output.MultipleRawFileOutputFormat',
              'sequencefile': 'fm.last.feathers.output.MultipleSequenceFiles'}
     else:
         outputformat_shortcuts = \
             {'code': 'org.apache.hadoop.mapred.SequenceFileOutputFormat',
              'text': 'org.apache.hadoop.mapred.TextOutputFormat',
              'raw': 'fm.last.feathers.output.RawFileOutputFormat',
              'sequencefile': 'org.apache.hadoop.mapred.SequenceFileOutputFormat'}
     outputformat_shortcuts.update(configopts('outputformats', self.prog))
     outputformat = addedopts['outputformat'][0]
     if outputformat_shortcuts.has_key(outputformat.lower()):
         outputformat = outputformat_shortcuts[outputformat.lower()]
     self.opts.append(('outputformat', outputformat))
     if addedopts['addpath'] and addedopts['addpath'][0] != 'no':
         self.opts.append(('cmdenv', 'dumbo_addpath=true'))
     pyenv = envdef('PYTHONPATH',
                    addedopts['libegg'],
                    'file',
                    self.opts,
                    shortcuts=dict(configopts('eggs', self.prog)),
                    quote=False,
                    trim=True,
                    extrapaths=addedopts['pypath'])
     if pyenv:
         self.opts.append(('cmdenv', pyenv))
     hadenv = envdef('HADOOP_CLASSPATH', addedopts['libjar'], 'libjar', 
                     self.opts, shortcuts=dict(configopts('jars', self.prog)))
     fileopt = getopt(self.opts, 'file')
     if fileopt:
         tmpfiles = []
         for file in fileopt:
             if file.startswith('file://'):
                 self.opts.append(('file', file[7:]))
             else:
                 tmpfiles.append(file)
         if tmpfiles:
             self.opts.append(('jobconf', 'tmpfiles=' + ','.join(tmpfiles)))
     libjaropt = getopt(self.opts, 'libjar')
     if libjaropt:
         tmpjars = []
         for jar in libjaropt:
             if jar.startswith('file://'):
                 self.opts.append(('file', jar[7:]))
             else:
                 tmpjars.append(jar)
         if tmpjars:
             self.opts.append(('jobconf', 'tmpjars=' + ','.join(tmpjars)))
     cmd = hadoop + '/bin/hadoop jar ' + streamingjar
     retval = execute(cmd, self.opts, hadenv)
     if addedopts['delinputs'] and addedopts['delinputs'][0] == 'yes':
         for (key, value) in self.opts:
             if key == 'input':
                 if os.path.exists(hadoop + "/bin/hdfs"):
                     hdfs = hadoop + "/bin/hdfs"
                 else:
                     hdfs = hadoop + "/bin/hadoop"
                 execute("%s dfs -rmr '%s'" % (hdfs, value))
     return retval
예제 #11
0
파일: unix.py 프로젝트: VickiFu/dumbo
    def run(self):
        retval = Iteration.run(self)
        if retval != 0:
            return retval
        addedopts = getopts(self.opts, ['input',
                                        'output',
                                        'mapper',
                                        'reducer',
                                        'libegg',
                                        'delinputs',
                                        'cmdenv',
                                        'pv',
                                        'addpath',
                                        'inputformat',
                                        'outputformat',
                                        'numreducetasks',
                                        'python',
                                        'pypath',
                                        'sorttmpdir',
                                        'sortbufsize'])
        (mapper, reducer) = (addedopts['mapper'][0], addedopts['reducer'][0])
        if not addedopts['input'] or not addedopts['output']:
            print >> sys.stderr, 'ERROR: input or output not specified'
            return 1
        inputs = reduce(operator.concat, (input.split(' ') for input in
                        addedopts['input']))
        output = addedopts['output'][0]
        pyenv = envdef('PYTHONPATH', addedopts['libegg'],
                       shortcuts=dict(configopts('eggs', self.prog)),
                       extrapaths=addedopts['pypath'])
        cmdenv = ' '.join("%s='%s'" % tuple(arg.split('=')) for arg in
                          addedopts['cmdenv'])
        if addedopts['pv'] and addedopts['pv'][0] == 'yes':
            mpv = '| pv -s `du -b %s | cut -f 1` -cN map ' % ' '.join(inputs)
            (spv, rpv) = ('| pv -cN sort ', '| pv -cN reduce ')
        else:
            (mpv, spv, rpv) = ('', '', '')

        (sorttmpdir, sortbufsize) = ('', '')
        if addedopts['sorttmpdir']:
            sorttmpdir = "-T %s" % addedopts['sorttmpdir'][0]
        if addedopts['sortbufsize']:
            sortbufsize = "-S %s" % addedopts['sortbufsize'][0]

        python = addedopts['python'][0]
        encodepipe = pyenv + ' ' + python + \
                     ' -m dumbo.cmd encodepipe -file ' + ' -file '.join(inputs)
        if addedopts['inputformat'] and addedopts['inputformat'][0] == 'code':
            encodepipe += ' -alreadycoded yes'
        if addedopts['addpath'] and addedopts['addpath'][0] != 'no':
            encodepipe += ' -addpath yes'
        if addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0':
            retval = execute("%s | %s %s %s %s > '%s'" % (encodepipe,
                                                          pyenv,
                                                          cmdenv,
                                                          mapper,
                                                          mpv,
                                                          output))
        else:
            retval = execute("%s | %s %s %s %s| LC_ALL=C sort %s %s %s| %s %s %s %s> '%s'"
                             % (encodepipe,
                                pyenv,
                                cmdenv,
                                mapper,
                                mpv,
                                sorttmpdir,
                                sortbufsize,
                                spv,
                                pyenv,
                                cmdenv,
                                reducer,
                                rpv,
                                output))
        if addedopts['delinputs'] and addedopts['delinputs'][0] == 'yes':
            for file in addedopts['input']:
                execute('rm ' + file)
        return retval
예제 #12
0
파일: streaming.py 프로젝트: CyaLiven/dumbo
    def run(self):
        retval = Iteration.run(self)
        if retval != 0:
            return retval
        opts = self.opts
        if os.path.exists(self.prog):
            opts.add('file', self.prog)

        keys = ['hadoop', 'name', 'delinputs', 'libegg', 'libjar',
            'inputformat', 'outputformat', 'nummaptasks', 'numreducetasks',
            'priority', 'queue', 'cachefile', 'cachearchive', 'file',
            'codewritable', 'addpath', 'getpath', 'python', 'streamoutput',
            'pypath', 'hadooplib']
        addedopts = opts.filter(keys)
        opts.remove(*keys)

        hadoop = findhadoop(addedopts['hadoop'][0])
        streamingjar = findjar(hadoop, 'streaming', addedopts['hadooplib'])
        if not streamingjar:
            print >> sys.stderr, 'ERROR: Streaming jar not found'
            return 1

        try:
            import typedbytes
        except ImportError:
            print >> sys.stderr, 'ERROR: "typedbytes" module not found'
            return 1
        modpath = re.sub('\.egg.*$', '.egg', typedbytes.__file__)
        if modpath.endswith('.egg'):
            addedopts.add('libegg', modpath)
        else:
            opts.add('file', modpath)
        opts.add('jobconf', 'stream.map.input=typedbytes')
        opts.add('jobconf', 'stream.reduce.input=typedbytes')

        if addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0':
            opts.add('jobconf', 'stream.reduce.output=typedbytes')
            if addedopts['streamoutput']:
                id_ = addedopts['streamoutput'][0]
                opts.add('jobconf', 'stream.map.output=' + id_)
            else:
                opts.add('jobconf', 'stream.map.output=typedbytes')
        else:
            opts.add('jobconf', 'stream.map.output=typedbytes')
            if addedopts['streamoutput']:
                id_ = addedopts['streamoutput'][0]
                opts.add('jobconf', 'stream.reduce.output=' + id_)
            else:
                opts.add('jobconf', 'stream.reduce.output=typedbytes')

        progname = self.prog.split('/')[-1] if not addedopts['name'] \
                                            else addedopts['name'][0]
        opts.add('jobconf', 'mapred.job.name=%s' % progname)

        nummaptasks = addedopts['nummaptasks']
        numreducetasks = addedopts['numreducetasks']
        if nummaptasks:
            opts.add('jobconf', 'mapred.map.tasks=%s' % nummaptasks[0])
        if numreducetasks:
            opts.add('numReduceTasks', numreducetasks[0])
        if addedopts['priority']:
            opts.add('jobconf', 'mapred.job.priority=%s' % addedopts['priority'][0])
        if addedopts['queue']:
            opts.add('jobconf', 'mapred.job.queue.name=%s' % addedopts['queue'][0])

        for cachefile in addedopts['cachefile']:
            opts.add('cacheFile', cachefile)

        for cachearchive in addedopts['cachearchive']:
            opts.add('cacheArchive', cachearchive)

        for _file in addedopts['file']:
            if not '://' in _file:
                if not os.path.exists(_file):
                    raise ValueError('file "%s" does not exist' % _file)
                _file = 'file://%s' % os.path.abspath(_file)
            opts.add('file', _file)

        if not addedopts['inputformat']:
            addedopts.add('inputformat', 'auto')

        inputformat_shortcuts = {
            'code': 'org.apache.hadoop.streaming.AutoInputFormat',
            'text': 'org.apache.hadoop.mapred.TextInputFormat',
            'sequencefile': 'org.apache.hadoop.streaming.AutoInputFormat',
            'auto': 'org.apache.hadoop.streaming.AutoInputFormat'
        }
        inputformat_shortcuts.update(configopts('inputformats', self.prog))

        inputformat = addedopts['inputformat'][0]
        if inputformat.lower() in inputformat_shortcuts:
            inputformat = inputformat_shortcuts[inputformat.lower()]
        opts.add('inputformat', inputformat)

        if not addedopts['outputformat']:
            addedopts.add('outputformat', 'sequencefile')

        if addedopts['getpath'] and 'no' not in addedopts['getpath']:
            outputformat_shortcuts = {
                'code': 'fm.last.feathers.output.MultipleSequenceFiles',
                'text': 'fm.last.feathers.output.MultipleTextFiles',
                'raw': 'fm.last.feathers.output.MultipleRawFileOutputFormat',
                'sequencefile': 'fm.last.feathers.output.MultipleSequenceFiles'
            }
        else:
            outputformat_shortcuts = {
                'code': 'org.apache.hadoop.mapred.SequenceFileOutputFormat',
                'text': 'org.apache.hadoop.mapred.TextOutputFormat',
                'raw': 'fm.last.feathers.output.RawFileOutputFormat',
                'sequencefile': 'org.apache.hadoop.mapred.SequenceFileOutputFormat'
            }
        outputformat_shortcuts.update(configopts('outputformats', self.prog))

        outputformat = addedopts['outputformat'][0]
        if outputformat.lower() in outputformat_shortcuts:
            outputformat = outputformat_shortcuts[outputformat.lower()]
        opts.add('outputformat', outputformat)

        if addedopts['addpath'] and 'no' not in addedopts['addpath']:
            opts.add('cmdenv', 'dumbo_addpath=true')

        pyenv = envdef('PYTHONPATH', addedopts['libegg'], 'file', self.opts,
            shortcuts=dict(configopts('eggs', self.prog)), quote=False, trim=True,
            extrapaths=addedopts['pypath'])
        if pyenv:
            opts.add('cmdenv', pyenv)

        hadenv = envdef('HADOOP_CLASSPATH', addedopts['libjar'], 'libjar',
            self.opts, shortcuts=dict(configopts('jars', self.prog)))

        tmpfiles = []
        for _file in opts.pop('file'):
            if _file.startswith('file://'):
                opts.add('file', _file[7:])
            else:
                tmpfiles.append(_file)
        if tmpfiles:
            opts.add('jobconf', 'tmpfiles=%s' % ','.join(tmpfiles))

        tmpjars = []
        for jar in opts.pop('libjar'):
            if jar.startswith('file://'):
                opts.add('file', jar[7:])
            else:
                tmpjars.append(jar)
        if tmpjars:
            opts.add('jobconf', 'tmpjars=%s' % ','.join(tmpjars))

        cmd = hadoop + '/bin/hadoop jar ' + streamingjar
        retval = execute(cmd, opts, hadenv)

        if 'yes' in addedopts['delinputs']:
            inputs = opts['input']
            for path in inputs:
                execute("%s/bin/hadoop fs -rmr '%s'" % (hadoop, path))
        return retval
예제 #13
0
파일: cmd.py 프로젝트: andrix/dumbo
def cat(path, opts):
    opts += Options(configopts('common'))
    opts += Options(configopts('cat'))
    return create_filesystem(opts).cat(path, opts)
예제 #14
0
파일: cmd.py 프로젝트: andrix/dumbo
def rm(path, opts):
    opts += Options(configopts('common'))
    opts += Options(configopts('rm'))
    return create_filesystem(opts).rm(path, opts)
예제 #15
0
파일: cmd.py 프로젝트: jso/dumbo
def exists(path, opts):
    opts = Options(opts)
    opts += Options(configopts('common'))
    opts += Options(configopts('exists'))
    return create_filesystem(opts).exists(path, opts)
예제 #16
0
파일: cmd.py 프로젝트: jso/dumbo
def cat(path, opts):
    opts = Options(opts)
    opts += Options(configopts('common'))
    opts += Options(configopts('cat'))
    return create_filesystem(opts).cat(path, opts)
 def __init__(self, prog, opts):
     Iteration.__init__(self, prog, opts)
     self.opts += configopts('unix', prog, self.opts)
예제 #18
0
파일: streaming.py 프로젝트: nbyloff/dumbo
 def run(self):
     retval = Iteration.run(self)
     if retval != 0:
         return retval
     if os.path.exists(self.prog):
         self.opts.append(("file", self.prog))
     addedopts = getopts(
         self.opts,
         [
             "hadoop",
             "name",
             "delinputs",
             "libegg",
             "libjar",
             "inputformat",
             "outputformat",
             "nummaptasks",
             "numreducetasks",
             "priority",
             "queue",
             "cachefile",
             "cachearchive",
             "file",
             "codewritable",
             "addpath",
             "getpath",
             "python",
             "streamoutput",
             "pypath",
         ],
     )
     hadoop = findhadoop(addedopts["hadoop"][0])
     streamingjar = findjar(hadoop, "streaming")
     if not streamingjar:
         print >> sys.stderr, "ERROR: Streaming jar not found"
         return 1
     try:
         import typedbytes
     except ImportError:
         print >> sys.stderr, 'ERROR: "typedbytes" module not found'
         return 1
     modpath = re.sub("\.egg.*$", ".egg", typedbytes.__file__)
     if modpath.endswith(".egg"):
         addedopts["libegg"].append(modpath)
     else:
         self.opts.append(("file", modpath))
     self.opts.append(("jobconf", "stream.map.input=typedbytes"))
     self.opts.append(("jobconf", "stream.reduce.input=typedbytes"))
     if addedopts["numreducetasks"] and addedopts["numreducetasks"][0] == "0":
         self.opts.append(("jobconf", "stream.reduce.output=typedbytes"))
         if addedopts["streamoutput"]:
             id_ = addedopts["streamoutput"][0]
             self.opts.append(("jobconf", "stream.map.output=" + id_))
         else:
             self.opts.append(("jobconf", "stream.map.output=typedbytes"))
     else:
         self.opts.append(("jobconf", "stream.map.output=typedbytes"))
         if addedopts["streamoutput"]:
             id_ = addedopts["streamoutput"][0]
             self.opts.append(("jobconf", "stream.reduce.output=" + id_))
         else:
             self.opts.append(("jobconf", "stream.reduce.output=typedbytes"))
     if not addedopts["name"]:
         self.opts.append(("jobconf", "mapred.job.name=" + self.prog.split("/")[-1]))
     else:
         self.opts.append(("jobconf", "mapred.job.name=%s" % addedopts["name"][0]))
     if addedopts["nummaptasks"]:
         self.opts.append(("jobconf", "mapred.map.tasks=%s" % addedopts["nummaptasks"][0]))
     if addedopts["numreducetasks"]:
         numreducetasks = int(addedopts["numreducetasks"][0])
         self.opts.append(("numReduceTasks", str(numreducetasks)))
     if addedopts["priority"]:
         self.opts.append(("jobconf", "mapred.job.priority=%s" % addedopts["priority"][0]))
     if addedopts["queue"]:
         self.opts.append(("jobconf", "mapred.job.queue.name=%s" % addedopts["queue"][0]))
     if addedopts["cachefile"]:
         for cachefile in addedopts["cachefile"]:
             self.opts.append(("cacheFile", cachefile))
     if addedopts["cachearchive"]:
         for cachearchive in addedopts["cachearchive"]:
             self.opts.append(("cacheArchive", cachearchive))
     if addedopts["file"]:
         for file in addedopts["file"]:
             if not "://" in file:
                 if not os.path.exists(file):
                     raise ValueError('file "' + file + '" does not exist')
                 file = "file://" + os.path.abspath(file)
             self.opts.append(("file", file))
     if not addedopts["inputformat"]:
         addedopts["inputformat"] = ["auto"]
     inputformat_shortcuts = {
         "code": "org.apache.hadoop.streaming.AutoInputFormat",
         "text": "org.apache.hadoop.mapred.TextInputFormat",
         "sequencefile": "org.apache.hadoop.streaming.AutoInputFormat",
         "auto": "org.apache.hadoop.streaming.AutoInputFormat",
     }
     inputformat_shortcuts.update(configopts("inputformats", self.prog))
     inputformat = addedopts["inputformat"][0]
     if inputformat_shortcuts.has_key(inputformat.lower()):
         inputformat = inputformat_shortcuts[inputformat.lower()]
     self.opts.append(("inputformat", inputformat))
     if not addedopts["outputformat"]:
         addedopts["outputformat"] = ["sequencefile"]
     if addedopts["getpath"] and addedopts["getpath"] != "no":
         outputformat_shortcuts = {
             "code": "fm.last.feathers.output.MultipleSequenceFiles",
             "text": "fm.last.feathers.output.MultipleTextFiles",
             "raw": "fm.last.feathers.output.MultipleRawFileOutputFormat",
             "sequencefile": "fm.last.feathers.output.MultipleSequenceFiles",
         }
     else:
         outputformat_shortcuts = {
             "code": "org.apache.hadoop.mapred.SequenceFileOutputFormat",
             "text": "org.apache.hadoop.mapred.TextOutputFormat",
             "raw": "fm.last.feathers.output.RawFileOutputFormat",
             "sequencefile": "org.apache.hadoop.mapred.SequenceFileOutputFormat",
         }
     outputformat_shortcuts.update(configopts("outputformats", self.prog))
     outputformat = addedopts["outputformat"][0]
     if outputformat_shortcuts.has_key(outputformat.lower()):
         outputformat = outputformat_shortcuts[outputformat.lower()]
     self.opts.append(("outputformat", outputformat))
     if addedopts["addpath"] and addedopts["addpath"][0] != "no":
         self.opts.append(("cmdenv", "dumbo_addpath=true"))
     pyenv = envdef(
         "PYTHONPATH",
         addedopts["libegg"],
         "file",
         self.opts,
         shortcuts=dict(configopts("eggs", self.prog)),
         quote=False,
         trim=True,
         extrapaths=addedopts["pypath"],
     )
     if pyenv:
         self.opts.append(("cmdenv", pyenv))
     hadenv = envdef(
         "HADOOP_CLASSPATH", addedopts["libjar"], "libjar", self.opts, shortcuts=dict(configopts("jars", self.prog))
     )
     fileopt = getopt(self.opts, "file")
     if fileopt:
         tmpfiles = []
         for file in fileopt:
             if file.startswith("file://"):
                 self.opts.append(("file", file[7:]))
             else:
                 tmpfiles.append(file)
         if tmpfiles:
             self.opts.append(("jobconf", "tmpfiles=" + ",".join(tmpfiles)))
     libjaropt = getopt(self.opts, "libjar")
     if libjaropt:
         tmpjars = []
         for jar in libjaropt:
             if jar.startswith("file://"):
                 self.opts.append(("file", jar[7:]))
             else:
                 tmpjars.append(jar)
         if tmpjars:
             self.opts.append(("jobconf", "tmpjars=" + ",".join(tmpjars)))
     cmd = hadoop + "/bin/hadoop jar " + streamingjar
     retval = execute(cmd, self.opts, hadenv)
     if addedopts["delinputs"] and addedopts["delinputs"][0] == "yes":
         for (key, value) in self.opts:
             if key == "input":
                 if os.path.exists(hadoop + "/bin/hdfs"):
                     hdfs = hadoop + "/bin/hdfs"
                 else:
                     hdfs = hadoop + "/bin/hadoop"
                 execute("%s dfs -rmr '%s'" % (hdfs, value))
     return retval
예제 #19
0
파일: cmd.py 프로젝트: andrix/dumbo
def ls(path, opts):
    opts += Options(configopts('common'))
    opts += Options(configopts('ls'))
    return create_filesystem(opts).ls(path, opts)
예제 #20
0
파일: cmd.py 프로젝트: jso/dumbo
def rm(path, opts):
    opts = Options(opts)
    opts += Options(configopts('common'))
    opts += Options(configopts('rm'))
    return create_filesystem(opts).rm(path, opts)
예제 #21
0
파일: cmd.py 프로젝트: andrix/dumbo
def get(path1, path2, opts):
    opts += Options(configopts('common'))
    opts += Options(configopts('get'))
    return create_filesystem(opts).get(path1, path2, opts)
예제 #22
0
파일: cmd.py 프로젝트: jso/dumbo
def get(path1, path2, opts):
    opts = Options(opts)
    opts += Options(configopts('common'))
    opts += Options(configopts('get'))
    return create_filesystem(opts).get(path1, path2, opts)
예제 #23
0
파일: cmd.py 프로젝트: andrix/dumbo
def ls(path, opts):
    opts += Options(configopts('common'))
    opts += Options(configopts('ls'))
    return create_filesystem(opts).ls(path, opts)
예제 #24
0
파일: cmd.py 프로젝트: soxofaan/dumbo
def put(path1, path2, opts):
    opts = Options(opts)
    opts += Options(configopts('common'))
    opts += Options(configopts('put'))
    return create_filesystem(opts).put(path1, path2, opts)
예제 #25
0
파일: streaming.py 프로젝트: CyaLiven/dumbo
 def __init__(self, prog, opts):
     Iteration.__init__(self, prog, opts)
     self.opts += Options(configopts('streaming', prog, self.opts))
     hadoop_streaming = 'streaming_%s' % self.opts['hadoop'][0]
     self.opts += Options(configopts(hadoop_streaming, prog, self.opts))
예제 #26
0
    def run(self):
        retval = Iteration.run(self)
        if retval != 0:
            return retval
        opts = self.opts
        if os.path.exists(self.prog):
            opts.add('file', self.prog)

        keys = [
            'hadoop', 'name', 'delinputs', 'libegg', 'libjar', 'inputformat',
            'outputformat', 'nummaptasks', 'numreducetasks', 'priority',
            'queue', 'cachefile', 'cachearchive', 'file', 'codewritable',
            'addpath', 'getpath', 'python', 'streamoutput', 'pypath',
            'hadooplib'
        ]
        addedopts = opts.filter(keys)
        opts.remove(*keys)

        hadoop = findhadoop(addedopts['hadoop'][0])
        streamingjar = findjar(hadoop, 'streaming', addedopts['hadooplib'])
        if not streamingjar:
            print >> sys.stderr, 'ERROR: Streaming jar not found'
            return 1

        try:
            import typedbytes
        except ImportError:
            print >> sys.stderr, 'ERROR: "typedbytes" module not found'
            return 1
        modpath = re.sub('\.egg.*$', '.egg', typedbytes.__file__)
        if modpath.endswith('.egg'):
            addedopts.add('libegg', modpath)
        else:
            opts.add('file', modpath)
        opts.add('jobconf', 'stream.map.input=typedbytes')
        opts.add('jobconf', 'stream.reduce.input=typedbytes')

        if addedopts['numreducetasks'] and addedopts['numreducetasks'][
                0] == '0':
            opts.add('jobconf', 'stream.reduce.output=typedbytes')
            if addedopts['streamoutput']:
                id_ = addedopts['streamoutput'][0]
                opts.add('jobconf', 'stream.map.output=' + id_)
            else:
                opts.add('jobconf', 'stream.map.output=typedbytes')
        else:
            opts.add('jobconf', 'stream.map.output=typedbytes')
            if addedopts['streamoutput']:
                id_ = addedopts['streamoutput'][0]
                opts.add('jobconf', 'stream.reduce.output=' + id_)
            else:
                opts.add('jobconf', 'stream.reduce.output=typedbytes')

        progname = self.prog.split('/')[-1] if not addedopts['name'] \
                                            else addedopts['name'][0]
        opts.add('jobconf', 'mapred.job.name=%s' % progname)

        nummaptasks = addedopts['nummaptasks']
        numreducetasks = addedopts['numreducetasks']
        if nummaptasks:
            opts.add('jobconf', 'mapred.map.tasks=%s' % nummaptasks[0])
        if numreducetasks:
            opts.add('numReduceTasks', numreducetasks[0])
        if addedopts['priority']:
            opts.add('jobconf',
                     'mapred.job.priority=%s' % addedopts['priority'][0])
        if addedopts['queue']:
            opts.add('jobconf',
                     'mapred.job.queue.name=%s' % addedopts['queue'][0])

        for cachefile in addedopts['cachefile']:
            opts.add('cacheFile', cachefile)

        for cachearchive in addedopts['cachearchive']:
            opts.add('cacheArchive', cachearchive)

        for _file in addedopts['file']:
            if not '://' in _file:
                if not os.path.exists(_file):
                    raise ValueError('file "%s" does not exist' % _file)
                _file = 'file://%s' % os.path.abspath(_file)
            opts.add('file', _file)

        if not addedopts['inputformat']:
            addedopts.add('inputformat', 'auto')

        inputformat_shortcuts = {
            'code': 'org.apache.hadoop.streaming.AutoInputFormat',
            'text': 'org.apache.hadoop.mapred.TextInputFormat',
            'sequencefile': 'org.apache.hadoop.streaming.AutoInputFormat',
            'auto': 'org.apache.hadoop.streaming.AutoInputFormat'
        }
        inputformat_shortcuts.update(configopts('inputformats', self.prog))

        inputformat = addedopts['inputformat'][0]
        if inputformat.lower() in inputformat_shortcuts:
            inputformat = inputformat_shortcuts[inputformat.lower()]
        opts.add('inputformat', inputformat)

        if not addedopts['outputformat']:
            addedopts.add('outputformat', 'sequencefile')

        if addedopts['getpath'] and 'no' not in addedopts['getpath']:
            outputformat_shortcuts = {
                'code': 'fm.last.feathers.output.MultipleSequenceFiles',
                'text': 'fm.last.feathers.output.MultipleTextFiles',
                'raw': 'fm.last.feathers.output.MultipleRawFileOutputFormat',
                'sequencefile': 'fm.last.feathers.output.MultipleSequenceFiles'
            }
        else:
            outputformat_shortcuts = {
                'code': 'org.apache.hadoop.mapred.SequenceFileOutputFormat',
                'text': 'org.apache.hadoop.mapred.TextOutputFormat',
                'raw': 'fm.last.feathers.output.RawFileOutputFormat',
                'sequencefile':
                'org.apache.hadoop.mapred.SequenceFileOutputFormat'
            }
        outputformat_shortcuts.update(configopts('outputformats', self.prog))

        outputformat = addedopts['outputformat'][0]
        if outputformat.lower() in outputformat_shortcuts:
            outputformat = outputformat_shortcuts[outputformat.lower()]
        opts.add('outputformat', outputformat)

        if addedopts['addpath'] and 'no' not in addedopts['addpath']:
            opts.add('cmdenv', 'dumbo_addpath=true')

        pyenv = envdef('PYTHONPATH',
                       addedopts['libegg'],
                       'file',
                       self.opts,
                       shortcuts=dict(configopts('eggs', self.prog)),
                       quote=False,
                       trim=True,
                       extrapaths=addedopts['pypath'])
        if pyenv:
            opts.add('cmdenv', pyenv)

        hadenv = envdef('HADOOP_CLASSPATH',
                        addedopts['libjar'],
                        'libjar',
                        self.opts,
                        shortcuts=dict(configopts('jars', self.prog)))

        tmpfiles = []
        for _file in opts.pop('file'):
            if _file.startswith('file://'):
                opts.add('file', _file[7:])
            else:
                tmpfiles.append(_file)
        if tmpfiles:
            opts.add('jobconf', 'tmpfiles=%s' % ','.join(tmpfiles))

        tmpjars = []
        for jar in opts.pop('libjar'):
            if jar.startswith('file://'):
                opts.add('file', jar[7:])
            else:
                tmpjars.append(jar)
        if tmpjars:
            opts.add('jobconf', 'tmpjars=%s' % ','.join(tmpjars))

        cmd = hadoop + '/bin/hadoop jar ' + streamingjar
        retval = execute(cmd, opts, hadenv)

        if 'yes' in addedopts['delinputs']:
            inputs = opts['input']
            for path in inputs:
                execute("%s/bin/hadoop fs -rmr '%s'" % (hadoop, path))
        return retval
예제 #27
0
파일: unix.py 프로젝트: VickiFu/dumbo
 def __init__(self, prog, opts):
     Iteration.__init__(self, prog, opts)
     self.opts += configopts('unix', prog, self.opts)
예제 #28
0
파일: streaming.py 프로젝트: dgleich/dumbo
 def __init__(self, prog, opts):
     Iteration.__init__(self, prog, opts)
     self.opts += configopts('streaming', prog, self.opts)
     hadoop = getopt(self.opts, 'hadoop', delete=False)[0]
     self.opts += configopts('streaming_' + hadoop, prog, self.opts)
예제 #29
0
 def __init__(self, prog, opts):
     Iteration.__init__(self, prog, opts)
     self.opts += Options(configopts('streaming', prog, self.opts))
     hadoop_streaming = 'streaming_%s' % self.opts['hadoop'][0]
     self.opts += Options(configopts(hadoop_streaming, prog, self.opts))
예제 #30
0
 def __init__(self, prog, opts):
     Iteration.__init__(self, prog, opts)
     self.opts += configopts('streaming', prog, self.opts)
     hadoop = getopt(self.opts, 'hadoop', delete=False)[0]
     self.opts += configopts('streaming_' + hadoop, prog, self.opts)
예제 #31
0
파일: punix.py 프로젝트: jso/dumbo
    def run(self):
        retval = Iteration.run(self)
        if retval != 0:
            return retval
        addedopts = getopts(self.opts, ['input',
                                        'output',
                                        'mapper',
                                        'reducer',
                                        'libegg',
                                        'delinputs',
                                        'cmdenv',
                                        'inputformat',
                                        'outputformat',
                                        'numreducetasks',
                                        'python',
                                        'pypath',
                                        'tmpdir',
                                        'nmappers',
                                        'nreducers',
                                        'permapper',
                                        'shell'])
        (mapper, reducer) = (addedopts['mapper'][0], addedopts['reducer'][0])
        if not addedopts['input'] or not addedopts['output']:
            print >> sys.stderr, 'ERROR: input or output not specified'
            return 1
        inputs = reduce(operator.concat, (input.split(' ') for input in
                        addedopts['input']))
        output = addedopts['output'][0]
        try: os.makedirs(output)
        except os.error as e: pass

        pyenv = envdef('PYTHONPATH', addedopts['libegg'],
                       shortcuts=dict(configopts('eggs', self.prog)),
                       extrapaths=addedopts['pypath'])
        cmdenv = ' '.join("%s='%s'" % tuple(arg.split('=')) for arg in
                          addedopts['cmdenv'])

        shell = addedopts["shell"][0]

        python = addedopts['python'][0]

        mapTotal = len(inputs)
        mapDoneCount = [0]
        reduceDoneCount = [0]

        nMappers = int(addedopts["nmappers"][0])
        nReducers = int(addedopts["nreducers"][0])

        # this is the number of files that will be handed to each mapper
        permapper = int(addedopts["permapper"][0])

        # start the mappers, reducers
        mPool = Pool(nMappers)
        rPool = Pool(nReducers)

        doReduces = not (addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0')

        # set up the mapper output/reducer input directories
        tmpdir = os.sep.join([addedopts['tmpdir'][0], "%s_%06d" % (time.strftime("%Y-%m-%d_%H-%M-%S", time.gmtime()), random.randint(0, 999999))])

        mLock = threading.Lock()
        mResults = {}
        rLock = threading.Lock()
        mByR = {}
        rStarted = set()
        rResults = {}

        # start the map status output copier
        mapErrOutputCopier = MapErrOutputCopier(tmpdir)
        mapErrOutputCopier.start()

        if doReduces:
            # start the copy threads to handle map outputs
            copyLock = threading.Lock()
            copyThreads = {}
            for i in range(nReducers):
                copyThreads[i] = ReduceInputCopier(tmpdir, i)
                copyThreads[i].start()

            # start the reduce status output copier
            reduceErrOutputCopier = ReduceErrOutputCopier(tmpdir)
            reduceErrOutputCopier.start()

            for i in range(nReducers):
                try: os.makedirs(os.sep.join([tmpdir, "r-%d" % i]))
                except os.error as e: pass

                mByR[i] = set()

        # do maps -- kick it all off
        if permapper == 1:
            for args in enumerate(inputs):
                i, filename = args
                args = pyenv, python, cmdenv, mapper, nReducers, tmpdir, output, addedopts, shell, i, [filename], doReduces
                mLock.acquire()
                mResults[i] = mPool.apply_async(doMap, args)
                mLock.release()
        else:
            # multiple files per mapper...
            remaining = list(inputs)
            i = 0
            while remaining:
                args = pyenv, python, cmdenv, mapper, nReducers, tmpdir, output, addedopts, shell, i, remaining[:permapper], doReduces
                mLock.acquire()
                mResults[i] = mPool.apply_async(doMap, args)
                mLock.release()

                remaining = remaining[permapper:]
                i += 1
            
            # need to reset the mapTotal variable since we have fewer tasks...
            mapTotal = i

        def reduceDone():
            # did anything finish?
            rLock.acquire()

            done = [x for x in rResults if rResults[x].ready()]
            for args in done: del rResults[args] # cleanup

            rLock.release()

            for reducenum in done:
                #print "reduce %d done" % reducenum
                reduceDoneCount[0] += 1

                reduceErrOutputCopier.reduce_done(reducenum)

        def mapDone():
            # did anything finish?
            mLock.acquire()

            done = [x for x in mResults if mResults[x].ready()]
            for args in done: del mResults[args] # cleanup

            mLock.release()

            for args in done:
                i = args

                mapDoneCount[0] += 1

                mapErrOutputCopier.map_done(i)

                if doReduces:
                    #print "map %d done" % i

                    # update the structures
                    for reducenum in range(nReducers):
                        # initiate the copy request...
                        copyThreads[reducenum].map_done(i)

                        rLock.acquire()

                        mByR[reducenum].add(i)

                        # see if we can signal that's all the copier will have to handle?
                        if len(mByR[reducenum]) == mapTotal:
                            copyThreads[reducenum].map_done(None)

                        rLock.release()
                else:
                    # just move the map output file (unsorted) to the output directory
                    print "map %d done" % i


        def copyDone():
            # did anything finish?
            copyLock.acquire()

            done = [x for x in copyThreads if not copyThreads[x].is_alive()]

            for rnum in done: del copyThreads[rnum] # cleanup

            copyLock.release()

            for rnum in done:
                rLock.acquire()

                rStarted.add(rnum)
                args = tmpdir, pyenv, cmdenv, reducer, output, shell, rnum
                rResults[rnum] = rPool.apply_async(doReduce, args)

                rLock.release()

        while reduceDoneCount[0] < nReducers:
            # check for things finishing...
            mapDone()
            copyDone()
            reduceDone()

            mLock.acquire()
            haveMaps = len(mResults)
            mLock.release()

            rLock.acquire()
            haveReduces = len(rResults)
            rLock.release()

            copyLock.acquire()
            copyRunning = len(copyThreads)
            copyLock.release()

            print "%d/%d/%d maps\t%d/%d copies\t%d/%d/%d reduces" % (haveMaps, mapDoneCount[0], mapTotal, copyRunning, nReducers, haveReduces, reduceDoneCount[0], nReducers)

            time.sleep(5)

        mPool.terminate()
        mPool.join()
        rPool.terminate()
        rPool.join()

        # make sure the map status output is done before cleaning up the tmp dir
        mapErrOutputCopier.map_done(None)
        mapErrOutputCopier.join()

        if doReduces:
            # make sure the reduce status output is done before cleaning up the tmp dir
            reduceErrOutputCopier.reduce_done(None)
            reduceErrOutputCopier.join()

        if not master_debug and len(os.listdir(tmpdir)) == 0:
            os.rmdir(tmpdir)


        return 0 # make sure we return an error if there is a problem.
예제 #32
0
파일: unix.py 프로젝트: joskid/dumbo
 def __init__(self, prog, opts):
     Iteration.__init__(self, prog, opts)
     self.opts += Options(configopts("unix", prog, self.opts))
예제 #33
0
파일: streaming.py 프로젝트: dgleich/dumbo
 def run(self):
     retval = Iteration.run(self)
     if retval != 0:
         return retval
     if os.path.exists(self.prog):
         self.opts.append(('file', self.prog))
     addedopts = getopts(self.opts, ['hadoop',
                                     'name',
                                     'delinputs',
                                     'libegg',
                                     'libjar',
                                     'libjarstreaming',
                                     'inputformat',
                                     'outputformat',
                                     'nummaptasks',
                                     'numreducetasks',
                                     'priority',
                                     'queue',
                                     'cachefile',
                                     'cachearchive',
                                     'file',
                                     'codewritable',
                                     'addpath',
                                     'getpath',
                                     'python',
                                     'streamoutput',
                                     'pypath'])
     hadoop = findhadoop(addedopts['hadoop'][0])
     streamingjar = getopt(self.opts,'streamingjar')
     if streamingjar is None or len(streamingjar)==0:
         streamingjar = findjar(hadoop,'streaming')
     else:
         streamingjar = streamingjar[0]
     if not streamingjar:
         print >> sys.stderr, 'ERROR: Streaming jar not found'
         return 1
         
     # add typedbytes to path
     try: 
         import typedbytes
     except ImportError:
         print >> sys.stderr, 'ERROR: "typedbytes" module not found'
         return 1
     modpath = re.sub('\.egg.*$', '.egg', typedbytes.__file__)
     if modpath.endswith('.egg'):            
         addedopts['libegg'].append(modpath)    
     else:
         self.opts.append(('file', modpath)) 
         
     # add ctypedbytes to job
     try: 
         import ctypedbytes
         print >>sys.stderr, 'INFO: "ctypedbytes" found!'
         modpath = re.sub('\.egg.*$', '.egg', ctypedbytes.__file__)
         if modpath.endswith('.egg'):            
             addedopts['libegg'].append(modpath)
     except ImportError:
         pass        
         
         
     self.opts.append(('jobconf', 'stream.map.input=typedbytes'))
     self.opts.append(('jobconf', 'stream.reduce.input=typedbytes'))
     if addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0':
         self.opts.append(('jobconf', 'stream.reduce.output=typedbytes'))
         if addedopts['streamoutput']:
             id_ = addedopts['streamoutput'][0]
             self.opts.append(('jobconf', 'stream.map.output=' + id_))
         else: 
             self.opts.append(('jobconf', 'stream.map.output=typedbytes'))
     else:
         self.opts.append(('jobconf', 'stream.map.output=typedbytes'))
         if addedopts['streamoutput']:
             id_ = addedopts['streamoutput'][0]
             self.opts.append(('jobconf', 'stream.reduce.output=' + id_))
         else:
             self.opts.append(('jobconf', 'stream.reduce.output=typedbytes'))
     if not addedopts['name']:
         self.opts.append(('jobconf', 'mapred.job.name='
                           + self.prog.split('/')[-1]))
     else:
         self.opts.append(('jobconf', 'mapred.job.name=%s'
                           % addedopts['name'][0]))
     if addedopts['nummaptasks']:
         self.opts.append(('jobconf', 'mapred.map.tasks=%s'
                           % addedopts['nummaptasks'][0]))
     if addedopts['numreducetasks']:
         numreducetasks = int(addedopts['numreducetasks'][0])
         self.opts.append(('numReduceTasks', str(numreducetasks)))
     if addedopts['priority']:
         self.opts.append(('jobconf', 'mapred.job.priority=%s'
                           % addedopts['priority'][0]))
     if addedopts['queue']:
         self.opts.append(('jobconf', 'mapred.job.queue.name=%s'
                           % addedopts['queue'][0]))
     if addedopts['cachefile']:
         for cachefile in addedopts['cachefile']:
             self.opts.append(('cacheFile', cachefile))
     if addedopts['cachearchive']:
         for cachearchive in addedopts['cachearchive']:
             self.opts.append(('cacheArchive', cachearchive))
     if addedopts['file']:
         for file in addedopts['file']:
             if not '://' in file:
                 if not os.path.exists(file):
                     raise ValueError('file "' + file + '" does not exist')
                 file = 'file://' + os.path.abspath(file)
             self.opts.append(('file', file))
     if not addedopts['inputformat']:
         addedopts['inputformat'] = ['auto']
     inputformat_shortcuts = \
         {'code': 'org.apache.hadoop.streaming.AutoInputFormat',
          'text': 'org.apache.hadoop.mapred.TextInputFormat',
          'sequencefile': 'org.apache.hadoop.streaming.AutoInputFormat',
          'auto': 'org.apache.hadoop.streaming.AutoInputFormat'}
     inputformat_shortcuts.update(configopts('inputformats', self.prog))
     inputformat = addedopts['inputformat'][0]
     if inputformat_shortcuts.has_key(inputformat.lower()):
         inputformat = inputformat_shortcuts[inputformat.lower()]
     self.opts.append(('inputformat', inputformat))
     if not addedopts['outputformat']:
         addedopts['outputformat'] = ['sequencefile']
     if addedopts['getpath'] and addedopts['getpath'] != 'no':
         outputformat_shortcuts = \
             {'code': 'fm.last.feathers.output.MultipleSequenceFiles',
              'text': 'fm.last.feathers.output.MultipleTextFiles',               
              'raw': 'fm.last.feathers.output.MultipleRawFileOutputFormat',
              'sequencefile': 'fm.last.feathers.output.MultipleSequenceFiles'}
     else:
         outputformat_shortcuts = \
             {'code': 'org.apache.hadoop.mapred.SequenceFileOutputFormat',
              'text': 'org.apache.hadoop.mapred.TextOutputFormat',
              'raw': 'fm.last.feathers.output.RawFileOutputFormat',
              'sequencefile': 'org.apache.hadoop.mapred.SequenceFileOutputFormat'}
     outputformat_shortcuts.update(configopts('outputformats', self.prog))
     outputformat = addedopts['outputformat'][0]
     if outputformat_shortcuts.has_key(outputformat.lower()):
         outputformat = outputformat_shortcuts[outputformat.lower()]
     self.opts.append(('outputformat', outputformat))
     if addedopts['addpath'] and addedopts['addpath'][0] != 'no':
         self.opts.append(('cmdenv', 'dumbo_addpath=true'))
     pyenv = envdef('PYTHONPATH',
                    addedopts['libegg'],
                    'file',
                    self.opts,
                    shortcuts=dict(configopts('eggs', self.prog)),
                    quote=False,
                    trim=True,
                    extrapaths=addedopts['pypath'])
     if pyenv:
         self.opts.append(('cmdenv', pyenv))
     if addedopts['libjarstreaming'] and addedopts['libjarstreaming'][0] != 'no':
         addedopts['libjar'].append(streamingjar)
     hadenv = envdef('HADOOP_CLASSPATH', addedopts['libjar'], 'libjar', 
                     self.opts, shortcuts=dict(configopts('jars', self.prog)))
     fileopt = getopt(self.opts, 'file')
     if fileopt:
         tmpfiles = []
         for file in fileopt:
             if file.startswith('file://'):
                 self.opts.append(('file', file[7:]))
             else:
                 tmpfiles.append(file)
         if tmpfiles:
             self.opts.append(('jobconf', 'tmpfiles=' + ','.join(tmpfiles)))
     libjaropt = getopt(self.opts, 'libjar')
     if libjaropt:
         tmpjars = []
         for jar in libjaropt:
             if jar.startswith('file://'):
                 self.opts.append(('file', jar[7:]))
             else:
                 tmpjars.append(jar)
         if tmpjars:
             self.opts.append(('jobconf', 'tmpjars=' + ','.join(tmpjars)))
     cmd = hadoop + '/bin/hadoop jar ' + streamingjar
     retval = execute(cmd, self.opts, hadenv)
     if addedopts['delinputs'] and addedopts['delinputs'][0] == 'yes':
         for (key, value) in self.opts:
             if key == 'input':
                 if os.path.exists(hadoop + "/bin/hdfs"):
                     hdfs = hadoop + "/bin/hdfs"
                 else:
                     hdfs = hadoop + "/bin/hadoop"
                 execute("%s dfs -rmr '%s'" % (hdfs, value))
     return retval
예제 #34
0
파일: unix.py 프로젝트: joskid/dumbo
    def run(self):
        retval = Iteration.run(self)
        if retval != 0:
            return retval

        opts = self.opts
        keys = [
            "input",
            "output",
            "mapper",
            "reducer",
            "libegg",
            "delinputs",
            "cmdenv",
            "pv",
            "addpath",
            "inputformat",
            "outputformat",
            "numreducetasks",
            "python",
            "pypath",
            "sorttmpdir",
            "sortbufsize",
        ]
        addedopts = opts.filter(keys)
        opts.remove(*keys)

        mapper, reducer = addedopts["mapper"][0], addedopts["reducer"][0]
        if not addedopts["input"] or not addedopts["output"]:
            print >>sys.stderr, "ERROR: input or output not specified"
            return 1

        _inputs = addedopts["input"]
        _output = addedopts["output"]

        inputs = reduce(operator.concat, (inp.split(" ") for inp in _inputs))
        output = _output[0]

        pyenv = envdef(
            "PYTHONPATH",
            addedopts["libegg"],
            shortcuts=dict(configopts("eggs", self.prog)),
            extrapaths=addedopts["pypath"],
        )
        cmdenv = " ".join("%s='%s'" % tuple(arg.split("=")) for arg in addedopts["cmdenv"])

        if "yes" in addedopts["pv"]:
            mpv = "| pv -s `du -b %s | cut -f 1` -cN map " % " ".join(inputs)
            (spv, rpv) = ("| pv -cN sort ", "| pv -cN reduce ")
        else:
            (mpv, spv, rpv) = ("", "", "")

        sorttmpdir, sortbufsize = "", ""
        if addedopts["sorttmpdir"]:
            sorttmpdir = "-T %s" % addedopts["sorttmpdir"][0]
        if addedopts["sortbufsize"]:
            sortbufsize = "-S %s" % addedopts["sortbufsize"][0]

        python = addedopts["python"][0]
        encodepipe = pyenv + " " + python + " -m dumbo.cmd encodepipe -file " + " -file ".join(inputs)

        if "code" in addedopts["inputformat"]:
            encodepipe += " -alreadycoded yes"
        if addedopts["addpath"] and "no" not in addedopts["addpath"]:
            encodepipe += " -addpath yes"
        if addedopts["numreducetasks"] and addedopts["numreducetasks"][0] == "0":
            retval = execute("%s | %s %s %s %s > '%s'" % (encodepipe, pyenv, cmdenv, mapper, mpv, output))
        else:
            retval = execute(
                "%s | %s %s %s %s| LC_ALL=C sort %s %s %s| %s %s %s %s> '%s'"
                % (
                    encodepipe,
                    pyenv,
                    cmdenv,
                    mapper,
                    mpv,
                    sorttmpdir,
                    sortbufsize,
                    spv,
                    pyenv,
                    cmdenv,
                    reducer,
                    rpv,
                    output,
                )
            )

        if "yes" in addedopts["delinputs"]:
            for _file in addedopts["input"]:
                execute("rm " + _file)
        return retval
예제 #35
0
파일: streaming.py 프로젝트: nbyloff/dumbo
 def __init__(self, prog, opts):
     Iteration.__init__(self, prog, opts)
     self.opts += configopts("streaming", prog, self.opts)
     hadoop = getopt(self.opts, "hadoop", delete=False)[0]
     self.opts += configopts("streaming_" + hadoop, prog, self.opts)
예제 #36
0
파일: cmd.py 프로젝트: andrix/dumbo
def exists(path, opts):
    opts += Options(configopts('common'))
    opts += Options(configopts('exists'))
    return create_filesystem(opts).exists(path, opts)
예제 #37
0
파일: common.py 프로젝트: vkuznet/dumbo
    def run(self):
        opts = self.opts
        attrs = ['fake', 'debug', 'python', 'iteration', 'itercount', 'hadoop', 
            'starter', 'name', 'memlimit', 'param', 'parser', 'record', 
            'joinkeys', 'hadoopconf', 'mapper', 'reducer']
        addedopts = opts.filter(attrs)
        opts.remove(*attrs)

        if 'yes' in addedopts['fake']:
            def dummysystem(*args, **kwargs):
                return 0
            global system
            system = dummysystem  # not very clean, but it works...
        if 'yes' in addedopts['debug']:
            opts.add('cmdenv', 'dumbo_debug=yes')
        if not addedopts['python']:
            python = 'python'
        else:
            python = addedopts['python'][0]
        opts.add('python', python)
        if not addedopts['iteration']:
            iter = 0
        else:
            iter = int(addedopts['iteration'][0])
        if not addedopts['itercount']:
            itercnt = 1
        else:
            itercnt = int(addedopts['itercount'][0])
        if addedopts['name']:
            name = addedopts['name'][0]
        else:
            name = self.prog.split('/')[-1]
        opts.add('name', '%s (%s/%s)' % (name, iter + 1, itercnt))
        if not addedopts['hadoop']:
            pypath = '/'.join(self.prog.split('/')[:-1])
            if pypath:
                opts.add('pypath', pypath)
        else:
            opts.add('hadoop', addedopts['hadoop'][0])
        progmod = self.prog.split('/')[-1]
        progmod = progmod[:-3] if progmod.endswith('.py') else progmod
        memlim = ' 262144000'  # 250MB limit by default
        if addedopts['memlimit']:
            # Limit amount of memory. This supports syntax 
            # of the form '256m', '12g' etc.
            try:
                _memlim = int(addedopts['memlimit'][0][:-1])
                memlim = ' %i' % {
                    'g': 1073741824    * _memlim,
                    'm': 1048576       * _memlim,
                    'k': 1024          * _memlim,
                    'b': 1             * _memlim,
                }[addedopts['memlimit'][0][-1].lower()]
            except KeyError:
                # Assume specified in bytes by default
                memlim = ' ' + addedopts['memlimit'][0]

        if addedopts['mapper']:
            opts.add('mapper', addedopts['mapper'][0])
        else:
            opts.add('mapper', '%s -m %s map %i%s' % (python, progmod, iter, 
                memlim))
        if addedopts['reducer']:
            opts.add('reducer', addedopts['reducer'][0])
        else:
            opts.add('reducer', '%s -m %s red %i%s' % (python, progmod, 
                iter, memlim))
        for param in addedopts['param']:
            opts.add('cmdenv', param)
        if addedopts['parser'] and iter == 0:
            parser = addedopts['parser'][0]
            shortcuts = dict(configopts('parsers', self.prog))
            if parser in shortcuts:
                parser = shortcuts[parser]
            opts.add('cmdenv', 'dumbo_parser=' + parser)
        if addedopts['record'] and iter == 0:
            record = addedopts['record'][0]
            shortcuts = dict(configopts('records', self.prog))
            if record in shortcuts:
                record = shortcuts[record]
            opts.add('cmdenv', 'dumbo_record=' + record)
        if 'yes' in addedopts['joinkeys']:
            opts.add('cmdenv', 'dumbo_joinkeys=yes')
            opts.add('partitioner', 'org.apache.hadoop.mapred.lib.BinaryPartitioner')
            opts.add('jobconf', 'mapred.binary.partitioner.right.offset=-6')
        for hadoopconf in addedopts['hadoopconf']:
            opts.add('jobconf', hadoopconf)
        opts.add('libegg', re.sub('\.egg.*$', '.egg', __file__))
        return 0
예제 #38
0
파일: cmd.py 프로젝트: andrix/dumbo
def put(path1, path2, opts):
    opts += Options(configopts('common'))
    opts += Options(configopts('put'))
    return create_filesystem(opts).put(path1, path2, opts)