def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
addedopts = getopts(self.opts, [
'input', 'output', 'mapper', 'reducer', 'libegg', 'delinputs',
'cmdenv', 'pv', 'addpath', 'inputformat', 'outputformat',
'numreducetasks', 'python', 'pypath', 'sorttmpdir', 'sortbufsize'
])
(mapper, reducer) = (addedopts['mapper'][0], addedopts['reducer'][0])
if not addedopts['input'] or not addedopts['output']:
print >> sys.stderr, 'ERROR: input or output not specified'
return 1
inputs = reduce(operator.concat,
(input.split(' ') for input in addedopts['input']))
output = addedopts['output'][0]
pyenv = envdef('PYTHONPATH',
addedopts['libegg'],
shortcuts=dict(configopts('eggs', self.prog)),
extrapaths=addedopts['pypath'])
cmdenv = ' '.join("%s='%s'" % tuple(arg.split('='))
for arg in addedopts['cmdenv'])
if addedopts['pv'] and addedopts['pv'][0] == 'yes':
mpv = '| pv -s `du -b %s | cut -f 1` -cN map ' % ' '.join(inputs)
(spv, rpv) = ('| pv -cN sort ', '| pv -cN reduce ')
else:
(mpv, spv, rpv) = ('', '', '')
(sorttmpdir, sortbufsize) = ('', '')
if addedopts['sorttmpdir']:
sorttmpdir = "-T %s" % addedopts['sorttmpdir'][0]
if addedopts['sortbufsize']:
sortbufsize = "-S %s" % addedopts['sortbufsize'][0]
python = addedopts['python'][0]
encodepipe = pyenv + ' ' + python + \
' -m dumbo.cmd encodepipe -file ' + ' -file '.join(inputs)
if addedopts['inputformat'] and addedopts['inputformat'][0] == 'code':
encodepipe += ' -alreadycoded yes'
if addedopts['addpath'] and addedopts['addpath'][0] != 'no':
encodepipe += ' -addpath yes'
if addedopts['numreducetasks'] and addedopts['numreducetasks'][
0] == '0':
retval = execute("%s | %s %s %s %s > '%s'" %
(encodepipe, pyenv, cmdenv, mapper, mpv, output))
else:
retval = execute(
"%s | %s %s %s %s| LC_ALL=C sort %s %s %s| %s %s %s %s> '%s'" %
(encodepipe, pyenv, cmdenv, mapper, mpv, sorttmpdir,
sortbufsize, spv, pyenv, cmdenv, reducer, rpv, output))
if addedopts['delinputs'] and addedopts['delinputs'][0] == 'yes':
for file in addedopts['input']:
execute('rm ' + file)
return retval
def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
if os.path.exists(self.prog):
self.opts.append(('file', self.prog))
addedopts = getopts(self.opts, ['hadoop',
'name',
'delinputs',
'libegg',
'libjar',
'libjarstreaming',
'inputformat',
'outputformat',
'nummaptasks',
'numreducetasks',
'priority',
'queue',
'cachefile',
'cachearchive',
'file',
'codewritable',
'addpath',
'getpath',
'python',
'streamoutput',
'pypath'])
hadoop = findhadoop(addedopts['hadoop'][0])
streamingjar = getopt(self.opts,'streamingjar')
if streamingjar is None or len(streamingjar)==0:
streamingjar = findjar(hadoop,'streaming')
else:
streamingjar = streamingjar[0]
if not streamingjar:
print >> sys.stderr, 'ERROR: Streaming jar not found'
return 1
# add typedbytes to path
try:
import typedbytes
except ImportError:
print >> sys.stderr, 'ERROR: "typedbytes" module not found'
return 1
modpath = re.sub('\.egg.*$', '.egg', typedbytes.__file__)
if modpath.endswith('.egg'):
addedopts['libegg'].append(modpath)
else:
self.opts.append(('file', modpath))
# add ctypedbytes to job
try:
import ctypedbytes
print >>sys.stderr, 'INFO: "ctypedbytes" found!'
modpath = re.sub('\.egg.*$', '.egg', ctypedbytes.__file__)
if modpath.endswith('.egg'):
addedopts['libegg'].append(modpath)
except ImportError:
pass
self.opts.append(('jobconf', 'stream.map.input=typedbytes'))
self.opts.append(('jobconf', 'stream.reduce.input=typedbytes'))
if addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0':
self.opts.append(('jobconf', 'stream.reduce.output=typedbytes'))
if addedopts['streamoutput']:
id_ = addedopts['streamoutput'][0]
self.opts.append(('jobconf', 'stream.map.output=' + id_))
else:
self.opts.append(('jobconf', 'stream.map.output=typedbytes'))
else:
self.opts.append(('jobconf', 'stream.map.output=typedbytes'))
if addedopts['streamoutput']:
id_ = addedopts['streamoutput'][0]
self.opts.append(('jobconf', 'stream.reduce.output=' + id_))
else:
self.opts.append(('jobconf', 'stream.reduce.output=typedbytes'))
if not addedopts['name']:
self.opts.append(('jobconf', 'mapred.job.name='
+ self.prog.split('/')[-1]))
else:
self.opts.append(('jobconf', 'mapred.job.name=%s'
% addedopts['name'][0]))
if addedopts['nummaptasks']:
self.opts.append(('jobconf', 'mapred.map.tasks=%s'
% addedopts['nummaptasks'][0]))
if addedopts['numreducetasks']:
numreducetasks = int(addedopts['numreducetasks'][0])
self.opts.append(('numReduceTasks', str(numreducetasks)))
if addedopts['priority']:
self.opts.append(('jobconf', 'mapred.job.priority=%s'
% addedopts['priority'][0]))
if addedopts['queue']:
self.opts.append(('jobconf', 'mapred.job.queue.name=%s'
% addedopts['queue'][0]))
if addedopts['cachefile']:
for cachefile in addedopts['cachefile']:
self.opts.append(('cacheFile', cachefile))
if addedopts['cachearchive']:
for cachearchive in addedopts['cachearchive']:
self.opts.append(('cacheArchive', cachearchive))
if addedopts['file']:
for file in addedopts['file']:
if not '://' in file:
if not os.path.exists(file):
raise ValueError('file "' + file + '" does not exist')
file = 'file://' + os.path.abspath(file)
self.opts.append(('file', file))
if not addedopts['inputformat']:
addedopts['inputformat'] = ['auto']
inputformat_shortcuts = \
{'code': 'org.apache.hadoop.streaming.AutoInputFormat',
'text': 'org.apache.hadoop.mapred.TextInputFormat',
'sequencefile': 'org.apache.hadoop.streaming.AutoInputFormat',
'auto': 'org.apache.hadoop.streaming.AutoInputFormat'}
inputformat_shortcuts.update(configopts('inputformats', self.prog))
inputformat = addedopts['inputformat'][0]
if inputformat_shortcuts.has_key(inputformat.lower()):
inputformat = inputformat_shortcuts[inputformat.lower()]
self.opts.append(('inputformat', inputformat))
if not addedopts['outputformat']:
addedopts['outputformat'] = ['sequencefile']
if addedopts['getpath'] and addedopts['getpath'] != 'no':
outputformat_shortcuts = \
{'code': 'fm.last.feathers.output.MultipleSequenceFiles',
'text': 'fm.last.feathers.output.MultipleTextFiles',
'raw': 'fm.last.feathers.output.MultipleRawFileOutputFormat',
'sequencefile': 'fm.last.feathers.output.MultipleSequenceFiles'}
else:
outputformat_shortcuts = \
{'code': 'org.apache.hadoop.mapred.SequenceFileOutputFormat',
'text': 'org.apache.hadoop.mapred.TextOutputFormat',
'raw': 'fm.last.feathers.output.RawFileOutputFormat',
'sequencefile': 'org.apache.hadoop.mapred.SequenceFileOutputFormat'}
outputformat_shortcuts.update(configopts('outputformats', self.prog))
outputformat = addedopts['outputformat'][0]
if outputformat_shortcuts.has_key(outputformat.lower()):
outputformat = outputformat_shortcuts[outputformat.lower()]
self.opts.append(('outputformat', outputformat))
if addedopts['addpath'] and addedopts['addpath'][0] != 'no':
self.opts.append(('cmdenv', 'dumbo_addpath=true'))
pyenv = envdef('PYTHONPATH',
addedopts['libegg'],
'file',
self.opts,
shortcuts=dict(configopts('eggs', self.prog)),
quote=False,
trim=True,
extrapaths=addedopts['pypath'])
if pyenv:
self.opts.append(('cmdenv', pyenv))
if addedopts['libjarstreaming'] and addedopts['libjarstreaming'][0] != 'no':
addedopts['libjar'].append(streamingjar)
hadenv = envdef('HADOOP_CLASSPATH', addedopts['libjar'], 'libjar',
self.opts, shortcuts=dict(configopts('jars', self.prog)))
fileopt = getopt(self.opts, 'file')
if fileopt:
tmpfiles = []
for file in fileopt:
if file.startswith('file://'):
self.opts.append(('file', file[7:]))
else:
tmpfiles.append(file)
if tmpfiles:
self.opts.append(('jobconf', 'tmpfiles=' + ','.join(tmpfiles)))
libjaropt = getopt(self.opts, 'libjar')
if libjaropt:
tmpjars = []
for jar in libjaropt:
if jar.startswith('file://'):
self.opts.append(('file', jar[7:]))
else:
tmpjars.append(jar)
if tmpjars:
self.opts.append(('jobconf', 'tmpjars=' + ','.join(tmpjars)))
cmd = hadoop + '/bin/hadoop jar ' + streamingjar
retval = execute(cmd, self.opts, hadenv)
if addedopts['delinputs'] and addedopts['delinputs'][0] == 'yes':
for (key, value) in self.opts:
if key == 'input':
if os.path.exists(hadoop + "/bin/hdfs"):
hdfs = hadoop + "/bin/hdfs"
else:
hdfs = hadoop + "/bin/hadoop"
execute("%s dfs -rmr '%s'" % (hdfs, value))
return retval
def __init__(self, prog, opts):
Iteration.__init__(self, prog, opts)
self.opts += configopts('streaming', prog, self.opts)
hadoop = getopt(self.opts, 'hadoop', delete=False)[0]
self.opts += configopts('streaming_' + hadoop, prog, self.opts)
def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
opts = self.opts
if os.path.exists(self.prog):
opts.add('file', self.prog)
keys = [
'hadoop', 'name', 'delinputs', 'libegg', 'libjar', 'inputformat',
'outputformat', 'nummaptasks', 'numreducetasks', 'priority',
'queue', 'cachefile', 'cachearchive', 'file', 'codewritable',
'addpath', 'getpath', 'python', 'streamoutput', 'pypath',
'hadooplib'
]
addedopts = opts.filter(keys)
opts.remove(*keys)
hadoop = findhadoop(addedopts['hadoop'][0])
streamingjar = findjar(hadoop, 'streaming', addedopts['hadooplib'])
if not streamingjar:
print >> sys.stderr, 'ERROR: Streaming jar not found'
return 1
try:
import typedbytes
except ImportError:
print >> sys.stderr, 'ERROR: "typedbytes" module not found'
return 1
modpath = re.sub('\.egg.*$', '.egg', typedbytes.__file__)
if modpath.endswith('.egg'):
addedopts.add('libegg', modpath)
else:
opts.add('file', modpath)
opts.add('jobconf', 'stream.map.input=typedbytes')
opts.add('jobconf', 'stream.reduce.input=typedbytes')
if addedopts['numreducetasks'] and addedopts['numreducetasks'][
0] == '0':
opts.add('jobconf', 'stream.reduce.output=typedbytes')
if addedopts['streamoutput']:
id_ = addedopts['streamoutput'][0]
opts.add('jobconf', 'stream.map.output=' + id_)
else:
opts.add('jobconf', 'stream.map.output=typedbytes')
else:
opts.add('jobconf', 'stream.map.output=typedbytes')
if addedopts['streamoutput']:
id_ = addedopts['streamoutput'][0]
opts.add('jobconf', 'stream.reduce.output=' + id_)
else:
opts.add('jobconf', 'stream.reduce.output=typedbytes')
progname = self.prog.split('/')[-1] if not addedopts['name'] \
else addedopts['name'][0]
opts.add('jobconf', 'mapred.job.name=%s' % progname)
nummaptasks = addedopts['nummaptasks']
numreducetasks = addedopts['numreducetasks']
if nummaptasks:
opts.add('jobconf', 'mapred.map.tasks=%s' % nummaptasks[0])
if numreducetasks:
opts.add('numReduceTasks', numreducetasks[0])
if addedopts['priority']:
opts.add('jobconf',
'mapred.job.priority=%s' % addedopts['priority'][0])
if addedopts['queue']:
opts.add('jobconf',
'mapred.job.queue.name=%s' % addedopts['queue'][0])
for cachefile in addedopts['cachefile']:
opts.add('cacheFile', cachefile)
for cachearchive in addedopts['cachearchive']:
opts.add('cacheArchive', cachearchive)
for _file in addedopts['file']:
if not '://' in _file:
if not os.path.exists(_file):
raise ValueError('file "%s" does not exist' % _file)
_file = 'file://%s' % os.path.abspath(_file)
opts.add('file', _file)
if not addedopts['inputformat']:
addedopts.add('inputformat', 'auto')
inputformat_shortcuts = {
'code': 'org.apache.hadoop.streaming.AutoInputFormat',
'text': 'org.apache.hadoop.mapred.TextInputFormat',
'sequencefile': 'org.apache.hadoop.streaming.AutoInputFormat',
'auto': 'org.apache.hadoop.streaming.AutoInputFormat'
}
inputformat_shortcuts.update(configopts('inputformats', self.prog))
inputformat = addedopts['inputformat'][0]
if inputformat.lower() in inputformat_shortcuts:
inputformat = inputformat_shortcuts[inputformat.lower()]
opts.add('inputformat', inputformat)
if not addedopts['outputformat']:
addedopts.add('outputformat', 'sequencefile')
if addedopts['getpath'] and 'no' not in addedopts['getpath']:
outputformat_shortcuts = {
'code': 'fm.last.feathers.output.MultipleSequenceFiles',
'text': 'fm.last.feathers.output.MultipleTextFiles',
'raw': 'fm.last.feathers.output.MultipleRawFileOutputFormat',
'sequencefile': 'fm.last.feathers.output.MultipleSequenceFiles'
}
else:
outputformat_shortcuts = {
'code': 'org.apache.hadoop.mapred.SequenceFileOutputFormat',
'text': 'org.apache.hadoop.mapred.TextOutputFormat',
'raw': 'fm.last.feathers.output.RawFileOutputFormat',
'sequencefile':
'org.apache.hadoop.mapred.SequenceFileOutputFormat'
}
outputformat_shortcuts.update(configopts('outputformats', self.prog))
outputformat = addedopts['outputformat'][0]
if outputformat.lower() in outputformat_shortcuts:
outputformat = outputformat_shortcuts[outputformat.lower()]
opts.add('outputformat', outputformat)
if addedopts['addpath'] and 'no' not in addedopts['addpath']:
opts.add('cmdenv', 'dumbo_addpath=true')
pyenv = envdef('PYTHONPATH',
addedopts['libegg'],
'file',
self.opts,
shortcuts=dict(configopts('eggs', self.prog)),
quote=False,
trim=True,
extrapaths=addedopts['pypath'])
if pyenv:
opts.add('cmdenv', pyenv)
hadenv = envdef('HADOOP_CLASSPATH',
addedopts['libjar'],
'libjar',
self.opts,
shortcuts=dict(configopts('jars', self.prog)))
tmpfiles = []
for _file in opts.pop('file'):
if _file.startswith('file://'):
opts.add('file', _file[7:])
else:
tmpfiles.append(_file)
if tmpfiles:
opts.add('jobconf', 'tmpfiles=%s' % ','.join(tmpfiles))
tmpjars = []
for jar in opts.pop('libjar'):
if jar.startswith('file://'):
opts.add('file', jar[7:])
else:
tmpjars.append(jar)
if tmpjars:
opts.add('jobconf', 'tmpjars=%s' % ','.join(tmpjars))
cmd = hadoop + '/bin/hadoop jar ' + streamingjar
retval = execute(cmd, opts, hadenv)
if 'yes' in addedopts['delinputs']:
inputs = opts['input']
for path in inputs:
execute("%s/bin/hadoop fs -rmr '%s'" % (hadoop, path))
return retval
def __init__(self, prog, opts):
Iteration.__init__(self, prog, opts)
self.opts += Options(configopts('streaming', prog, self.opts))
hadoop_streaming = 'streaming_%s' % self.opts['hadoop'][0]
self.opts += Options(configopts(hadoop_streaming, prog, self.opts))
def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
addedopts = getopts(self.opts, ['input',
'output',
'mapper',
'reducer',
'libegg',
'delinputs',
'cmdenv',
'pv',
'addpath',
'inputformat',
'outputformat',
'numreducetasks',
'python',
'pypath',
'sorttmpdir',
'sortbufsize'])
(mapper, reducer) = (addedopts['mapper'][0], addedopts['reducer'][0])
if not addedopts['input'] or not addedopts['output']:
print >> sys.stderr, 'ERROR: input or output not specified'
return 1
inputs = reduce(operator.concat, (input.split(' ') for input in
addedopts['input']))
output = addedopts['output'][0]
pyenv = envdef('PYTHONPATH', addedopts['libegg'],
shortcuts=dict(configopts('eggs', self.prog)),
extrapaths=addedopts['pypath'])
cmdenv = ' '.join("%s='%s'" % tuple(arg.split('=')) for arg in
addedopts['cmdenv'])
if addedopts['pv'] and addedopts['pv'][0] == 'yes':
mpv = '| pv -s `du -b %s | cut -f 1` -cN map ' % ' '.join(inputs)
(spv, rpv) = ('| pv -cN sort ', '| pv -cN reduce ')
else:
(mpv, spv, rpv) = ('', '', '')
(sorttmpdir, sortbufsize) = ('', '')
if addedopts['sorttmpdir']:
sorttmpdir = "-T %s" % addedopts['sorttmpdir'][0]
if addedopts['sortbufsize']:
sortbufsize = "-S %s" % addedopts['sortbufsize'][0]
python = addedopts['python'][0]
encodepipe = pyenv + ' ' + python + \
' -m dumbo.cmd encodepipe -file ' + ' -file '.join(inputs)
if addedopts['inputformat'] and addedopts['inputformat'][0] == 'code':
encodepipe += ' -alreadycoded yes'
if addedopts['addpath'] and addedopts['addpath'][0] != 'no':
encodepipe += ' -addpath yes'
if addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0':
retval = execute("%s | %s %s %s %s > '%s'" % (encodepipe,
pyenv,
cmdenv,
mapper,
mpv,
output))
else:
retval = execute("%s | %s %s %s %s| LC_ALL=C sort %s %s %s| %s %s %s %s> '%s'"
% (encodepipe,
pyenv,
cmdenv,
mapper,
mpv,
sorttmpdir,
sortbufsize,
spv,
pyenv,
cmdenv,
reducer,
rpv,
output))
if addedopts['delinputs'] and addedopts['delinputs'][0] == 'yes':
for file in addedopts['input']:
execute('rm ' + file)
return retval
def __init__(self, prog, opts):
Iteration.__init__(self, prog, opts)
self.opts += configopts('unix', prog, self.opts)
def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
if os.path.exists(self.prog):
self.opts.append(('file', self.prog))
addedopts = getopts(self.opts, ['hadoop',
'name',
'delinputs',
'libegg',
'libjar',
'inputformat',
'outputformat',
'nummaptasks',
'numreducetasks',
'priority',
'queue',
'cachefile',
'cachearchive',
'file',
'codewritable',
'addpath',
'getpath',
'python',
'streamoutput',
'pypath'])
hadoop = findhadoop(addedopts['hadoop'][0])
streamingjar = findjar(hadoop, 'streaming')
if not streamingjar:
print >> sys.stderr, 'ERROR: Streaming jar not found'
return 1
try:
import typedbytes
except ImportError:
print >> sys.stderr, 'ERROR: "typedbytes" module not found'
return 1
modpath = re.sub('\.egg.*$', '.egg', typedbytes.__file__)
if modpath.endswith('.egg'):
addedopts['libegg'].append(modpath)
else:
self.opts.append(('file', modpath))
self.opts.append(('jobconf', 'stream.map.input=typedbytes'))
self.opts.append(('jobconf', 'stream.reduce.input=typedbytes'))
if addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0':
self.opts.append(('jobconf', 'stream.reduce.output=typedbytes'))
if addedopts['streamoutput']:
id_ = addedopts['streamoutput'][0]
self.opts.append(('jobconf', 'stream.map.output=' + id_))
else:
self.opts.append(('jobconf', 'stream.map.output=typedbytes'))
else:
self.opts.append(('jobconf', 'stream.map.output=typedbytes'))
if addedopts['streamoutput']:
id_ = addedopts['streamoutput'][0]
self.opts.append(('jobconf', 'stream.reduce.output=' + id_))
else:
self.opts.append(('jobconf', 'stream.reduce.output=typedbytes'))
if not addedopts['name']:
self.opts.append(('jobconf', 'mapred.job.name='
+ self.prog.split('/')[-1]))
else:
self.opts.append(('jobconf', 'mapred.job.name=%s'
% addedopts['name'][0]))
if addedopts['nummaptasks']:
self.opts.append(('jobconf', 'mapred.map.tasks=%s'
% addedopts['nummaptasks'][0]))
if addedopts['numreducetasks']:
numreducetasks = int(addedopts['numreducetasks'][0])
self.opts.append(('numReduceTasks', str(numreducetasks)))
if addedopts['priority']:
self.opts.append(('jobconf', 'mapred.job.priority=%s'
% addedopts['priority'][0]))
if addedopts['queue']:
self.opts.append(('jobconf', 'mapred.job.queue.name=%s'
% addedopts['queue'][0]))
if addedopts['cachefile']:
for cachefile in addedopts['cachefile']:
self.opts.append(('cacheFile', cachefile))
if addedopts['cachearchive']:
for cachearchive in addedopts['cachearchive']:
self.opts.append(('cacheArchive', cachearchive))
if addedopts['file']:
for file in addedopts['file']:
if not '://' in file:
if not os.path.exists(file):
raise ValueError('file "' + file + '" does not exist')
file = 'file://' + os.path.abspath(file)
self.opts.append(('file', file))
if not addedopts['inputformat']:
addedopts['inputformat'] = ['auto']
inputformat_shortcuts = \
{'code': 'org.apache.hadoop.streaming.AutoInputFormat',
'text': 'org.apache.hadoop.mapred.TextInputFormat',
'sequencefile': 'org.apache.hadoop.streaming.AutoInputFormat',
'auto': 'org.apache.hadoop.streaming.AutoInputFormat'}
inputformat_shortcuts.update(configopts('inputformats', self.prog))
inputformat = addedopts['inputformat'][0]
if inputformat_shortcuts.has_key(inputformat.lower()):
inputformat = inputformat_shortcuts[inputformat.lower()]
self.opts.append(('inputformat', inputformat))
if not addedopts['outputformat']:
addedopts['outputformat'] = ['sequencefile']
if addedopts['getpath'] and addedopts['getpath'] != 'no':
outputformat_shortcuts = \
{'code': 'fm.last.feathers.output.MultipleSequenceFiles',
'text': 'fm.last.feathers.output.MultipleTextFiles',
'raw': 'fm.last.feathers.output.MultipleRawFileOutputFormat',
'sequencefile': 'fm.last.feathers.output.MultipleSequenceFiles'}
else:
outputformat_shortcuts = \
{'code': 'org.apache.hadoop.mapred.SequenceFileOutputFormat',
'text': 'org.apache.hadoop.mapred.TextOutputFormat',
'raw': 'fm.last.feathers.output.RawFileOutputFormat',
'sequencefile': 'org.apache.hadoop.mapred.SequenceFileOutputFormat'}
outputformat_shortcuts.update(configopts('outputformats', self.prog))
outputformat = addedopts['outputformat'][0]
if outputformat_shortcuts.has_key(outputformat.lower()):
outputformat = outputformat_shortcuts[outputformat.lower()]
self.opts.append(('outputformat', outputformat))
if addedopts['addpath'] and addedopts['addpath'][0] != 'no':
self.opts.append(('cmdenv', 'dumbo_addpath=true'))
pyenv = envdef('PYTHONPATH',
addedopts['libegg'],
'file',
self.opts,
shortcuts=dict(configopts('eggs', self.prog)),
quote=False,
trim=True,
extrapaths=addedopts['pypath'])
if pyenv:
self.opts.append(('cmdenv', pyenv))
hadenv = envdef('HADOOP_CLASSPATH', addedopts['libjar'], 'libjar',
self.opts, shortcuts=dict(configopts('jars', self.prog)))
fileopt = getopt(self.opts, 'file')
if fileopt:
tmpfiles = []
for file in fileopt:
if file.startswith('file://'):
self.opts.append(('file', file[7:]))
else:
tmpfiles.append(file)
if tmpfiles:
self.opts.append(('jobconf', 'tmpfiles=' + ','.join(tmpfiles)))
libjaropt = getopt(self.opts, 'libjar')
if libjaropt:
tmpjars = []
for jar in libjaropt:
if jar.startswith('file://'):
self.opts.append(('file', jar[7:]))
else:
tmpjars.append(jar)
if tmpjars:
self.opts.append(('jobconf', 'tmpjars=' + ','.join(tmpjars)))
cmd = hadoop + '/bin/hadoop jar ' + streamingjar
retval = execute(cmd, self.opts, hadenv)
if addedopts['delinputs'] and addedopts['delinputs'][0] == 'yes':
for (key, value) in self.opts:
if key == 'input':
if os.path.exists(hadoop + "/bin/hdfs"):
hdfs = hadoop + "/bin/hdfs"
else:
hdfs = hadoop + "/bin/hadoop"
execute("%s dfs -rmr '%s'" % (hdfs, value))
return retval
def __init__(self, prog, opts):
Iteration.__init__(self, prog, opts)
self.opts += Options(configopts('streaming', prog, self.opts))
hadoop_streaming = 'streaming_%s' % self.opts['hadoop'][0]
self.opts += Options(configopts(hadoop_streaming, prog, self.opts))
def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
opts = self.opts
keys = [
"input",
"output",
"mapper",
"reducer",
"libegg",
"delinputs",
"cmdenv",
"pv",
"addpath",
"inputformat",
"outputformat",
"numreducetasks",
"python",
"pypath",
"sorttmpdir",
"sortbufsize",
]
addedopts = opts.filter(keys)
opts.remove(*keys)
mapper, reducer = addedopts["mapper"][0], addedopts["reducer"][0]
if not addedopts["input"] or not addedopts["output"]:
print >>sys.stderr, "ERROR: input or output not specified"
return 1
_inputs = addedopts["input"]
_output = addedopts["output"]
inputs = reduce(operator.concat, (inp.split(" ") for inp in _inputs))
output = _output[0]
pyenv = envdef(
"PYTHONPATH",
addedopts["libegg"],
shortcuts=dict(configopts("eggs", self.prog)),
extrapaths=addedopts["pypath"],
)
cmdenv = " ".join("%s='%s'" % tuple(arg.split("=")) for arg in addedopts["cmdenv"])
if "yes" in addedopts["pv"]:
mpv = "| pv -s `du -b %s | cut -f 1` -cN map " % " ".join(inputs)
(spv, rpv) = ("| pv -cN sort ", "| pv -cN reduce ")
else:
(mpv, spv, rpv) = ("", "", "")
sorttmpdir, sortbufsize = "", ""
if addedopts["sorttmpdir"]:
sorttmpdir = "-T %s" % addedopts["sorttmpdir"][0]
if addedopts["sortbufsize"]:
sortbufsize = "-S %s" % addedopts["sortbufsize"][0]
python = addedopts["python"][0]
encodepipe = pyenv + " " + python + " -m dumbo.cmd encodepipe -file " + " -file ".join(inputs)
if "code" in addedopts["inputformat"]:
encodepipe += " -alreadycoded yes"
if addedopts["addpath"] and "no" not in addedopts["addpath"]:
encodepipe += " -addpath yes"
if addedopts["numreducetasks"] and addedopts["numreducetasks"][0] == "0":
retval = execute("%s | %s %s %s %s > '%s'" % (encodepipe, pyenv, cmdenv, mapper, mpv, output))
else:
retval = execute(
"%s | %s %s %s %s| LC_ALL=C sort %s %s %s| %s %s %s %s> '%s'"
% (
encodepipe,
pyenv,
cmdenv,
mapper,
mpv,
sorttmpdir,
sortbufsize,
spv,
pyenv,
cmdenv,
reducer,
rpv,
output,
)
)
if "yes" in addedopts["delinputs"]:
for _file in addedopts["input"]:
execute("rm " + _file)
return retval
def __init__(self, prog, opts):
Iteration.__init__(self, prog, opts)
self.opts += Options(configopts("unix", prog, self.opts))
def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
addedopts = getopts(self.opts, ['input',
'output',
'mapper',
'reducer',
'libegg',
'delinputs',
'cmdenv',
'inputformat',
'outputformat',
'numreducetasks',
'python',
'pypath',
'tmpdir',
'nmappers',
'nreducers',
'permapper',
'shell'])
(mapper, reducer) = (addedopts['mapper'][0], addedopts['reducer'][0])
if not addedopts['input'] or not addedopts['output']:
print >> sys.stderr, 'ERROR: input or output not specified'
return 1
inputs = reduce(operator.concat, (input.split(' ') for input in
addedopts['input']))
output = addedopts['output'][0]
try: os.makedirs(output)
except os.error as e: pass
pyenv = envdef('PYTHONPATH', addedopts['libegg'],
shortcuts=dict(configopts('eggs', self.prog)),
extrapaths=addedopts['pypath'])
cmdenv = ' '.join("%s='%s'" % tuple(arg.split('=')) for arg in
addedopts['cmdenv'])
shell = addedopts["shell"][0]
python = addedopts['python'][0]
mapTotal = len(inputs)
mapDoneCount = [0]
reduceDoneCount = [0]
nMappers = int(addedopts["nmappers"][0])
nReducers = int(addedopts["nreducers"][0])
# this is the number of files that will be handed to each mapper
permapper = int(addedopts["permapper"][0])
# start the mappers, reducers
mPool = Pool(nMappers)
rPool = Pool(nReducers)
doReduces = not (addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0')
# set up the mapper output/reducer input directories
tmpdir = os.sep.join([addedopts['tmpdir'][0], "%s_%06d" % (time.strftime("%Y-%m-%d_%H-%M-%S", time.gmtime()), random.randint(0, 999999))])
mLock = threading.Lock()
mResults = {}
rLock = threading.Lock()
mByR = {}
rStarted = set()
rResults = {}
# start the map status output copier
mapErrOutputCopier = MapErrOutputCopier(tmpdir)
mapErrOutputCopier.start()
if doReduces:
# start the copy threads to handle map outputs
copyLock = threading.Lock()
copyThreads = {}
for i in range(nReducers):
copyThreads[i] = ReduceInputCopier(tmpdir, i)
copyThreads[i].start()
# start the reduce status output copier
reduceErrOutputCopier = ReduceErrOutputCopier(tmpdir)
reduceErrOutputCopier.start()
for i in range(nReducers):
try: os.makedirs(os.sep.join([tmpdir, "r-%d" % i]))
except os.error as e: pass
mByR[i] = set()
# do maps -- kick it all off
if permapper == 1:
for args in enumerate(inputs):
i, filename = args
args = pyenv, python, cmdenv, mapper, nReducers, tmpdir, output, addedopts, shell, i, [filename], doReduces
mLock.acquire()
mResults[i] = mPool.apply_async(doMap, args)
mLock.release()
else:
# multiple files per mapper...
remaining = list(inputs)
i = 0
while remaining:
args = pyenv, python, cmdenv, mapper, nReducers, tmpdir, output, addedopts, shell, i, remaining[:permapper], doReduces
mLock.acquire()
mResults[i] = mPool.apply_async(doMap, args)
mLock.release()
remaining = remaining[permapper:]
i += 1
# need to reset the mapTotal variable since we have fewer tasks...
mapTotal = i
def reduceDone():
# did anything finish?
rLock.acquire()
done = [x for x in rResults if rResults[x].ready()]
for args in done: del rResults[args] # cleanup
rLock.release()
for reducenum in done:
#print "reduce %d done" % reducenum
reduceDoneCount[0] += 1
reduceErrOutputCopier.reduce_done(reducenum)
def mapDone():
# did anything finish?
mLock.acquire()
done = [x for x in mResults if mResults[x].ready()]
for args in done: del mResults[args] # cleanup
mLock.release()
for args in done:
i = args
mapDoneCount[0] += 1
mapErrOutputCopier.map_done(i)
if doReduces:
#print "map %d done" % i
# update the structures
for reducenum in range(nReducers):
# initiate the copy request...
copyThreads[reducenum].map_done(i)
rLock.acquire()
mByR[reducenum].add(i)
# see if we can signal that's all the copier will have to handle?
if len(mByR[reducenum]) == mapTotal:
copyThreads[reducenum].map_done(None)
rLock.release()
else:
# just move the map output file (unsorted) to the output directory
print "map %d done" % i
def copyDone():
# did anything finish?
copyLock.acquire()
done = [x for x in copyThreads if not copyThreads[x].is_alive()]
for rnum in done: del copyThreads[rnum] # cleanup
copyLock.release()
for rnum in done:
rLock.acquire()
rStarted.add(rnum)
args = tmpdir, pyenv, cmdenv, reducer, output, shell, rnum
rResults[rnum] = rPool.apply_async(doReduce, args)
rLock.release()
while reduceDoneCount[0] < nReducers:
# check for things finishing...
mapDone()
copyDone()
reduceDone()
mLock.acquire()
haveMaps = len(mResults)
mLock.release()
rLock.acquire()
haveReduces = len(rResults)
rLock.release()
copyLock.acquire()
copyRunning = len(copyThreads)
copyLock.release()
print "%d/%d/%d maps\t%d/%d copies\t%d/%d/%d reduces" % (haveMaps, mapDoneCount[0], mapTotal, copyRunning, nReducers, haveReduces, reduceDoneCount[0], nReducers)
time.sleep(5)
mPool.terminate()
mPool.join()
rPool.terminate()
rPool.join()
# make sure the map status output is done before cleaning up the tmp dir
mapErrOutputCopier.map_done(None)
mapErrOutputCopier.join()
if doReduces:
# make sure the reduce status output is done before cleaning up the tmp dir
reduceErrOutputCopier.reduce_done(None)
reduceErrOutputCopier.join()
if not master_debug and len(os.listdir(tmpdir)) == 0:
os.rmdir(tmpdir)
return 0 # make sure we return an error if there is a problem.
def __init__(self, prog, opts):
Iteration.__init__(self, prog, opts)
self.opts += configopts('unix', prog, self.opts)
def __init__(self, prog, opts):
Iteration.__init__(self, prog, opts)
self.opts += configopts("streaming", prog, self.opts)
hadoop = getopt(self.opts, "hadoop", delete=False)[0]
self.opts += configopts("streaming_" + hadoop, prog, self.opts)
def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
opts = self.opts
if os.path.exists(self.prog):
opts.add('file', self.prog)
keys = ['hadoop', 'name', 'delinputs', 'libegg', 'libjar',
'inputformat', 'outputformat', 'nummaptasks', 'numreducetasks',
'priority', 'queue', 'cachefile', 'cachearchive', 'file',
'codewritable', 'addpath', 'getpath', 'python', 'streamoutput',
'pypath', 'hadooplib']
addedopts = opts.filter(keys)
opts.remove(*keys)
hadoop = findhadoop(addedopts['hadoop'][0])
streamingjar = findjar(hadoop, 'streaming', addedopts['hadooplib'])
if not streamingjar:
print >> sys.stderr, 'ERROR: Streaming jar not found'
return 1
try:
import typedbytes
except ImportError:
print >> sys.stderr, 'ERROR: "typedbytes" module not found'
return 1
modpath = re.sub('\.egg.*$', '.egg', typedbytes.__file__)
if modpath.endswith('.egg'):
addedopts.add('libegg', modpath)
else:
opts.add('file', modpath)
opts.add('jobconf', 'stream.map.input=typedbytes')
opts.add('jobconf', 'stream.reduce.input=typedbytes')
if addedopts['numreducetasks'] and addedopts['numreducetasks'][0] == '0':
opts.add('jobconf', 'stream.reduce.output=typedbytes')
if addedopts['streamoutput']:
id_ = addedopts['streamoutput'][0]
opts.add('jobconf', 'stream.map.output=' + id_)
else:
opts.add('jobconf', 'stream.map.output=typedbytes')
else:
opts.add('jobconf', 'stream.map.output=typedbytes')
if addedopts['streamoutput']:
id_ = addedopts['streamoutput'][0]
opts.add('jobconf', 'stream.reduce.output=' + id_)
else:
opts.add('jobconf', 'stream.reduce.output=typedbytes')
progname = self.prog.split('/')[-1] if not addedopts['name'] \
else addedopts['name'][0]
opts.add('jobconf', 'mapred.job.name=%s' % progname)
nummaptasks = addedopts['nummaptasks']
numreducetasks = addedopts['numreducetasks']
if nummaptasks:
opts.add('jobconf', 'mapred.map.tasks=%s' % nummaptasks[0])
if numreducetasks:
opts.add('numReduceTasks', numreducetasks[0])
if addedopts['priority']:
opts.add('jobconf', 'mapred.job.priority=%s' % addedopts['priority'][0])
if addedopts['queue']:
opts.add('jobconf', 'mapred.job.queue.name=%s' % addedopts['queue'][0])
for cachefile in addedopts['cachefile']:
opts.add('cacheFile', cachefile)
for cachearchive in addedopts['cachearchive']:
opts.add('cacheArchive', cachearchive)
for _file in addedopts['file']:
if not '://' in _file:
if not os.path.exists(_file):
raise ValueError('file "%s" does not exist' % _file)
_file = 'file://%s' % os.path.abspath(_file)
opts.add('file', _file)
if not addedopts['inputformat']:
addedopts.add('inputformat', 'auto')
inputformat_shortcuts = {
'code': 'org.apache.hadoop.streaming.AutoInputFormat',
'text': 'org.apache.hadoop.mapred.TextInputFormat',
'sequencefile': 'org.apache.hadoop.streaming.AutoInputFormat',
'auto': 'org.apache.hadoop.streaming.AutoInputFormat'
}
inputformat_shortcuts.update(configopts('inputformats', self.prog))
inputformat = addedopts['inputformat'][0]
if inputformat.lower() in inputformat_shortcuts:
inputformat = inputformat_shortcuts[inputformat.lower()]
opts.add('inputformat', inputformat)
if not addedopts['outputformat']:
addedopts.add('outputformat', 'sequencefile')
if addedopts['getpath'] and 'no' not in addedopts['getpath']:
outputformat_shortcuts = {
'code': 'fm.last.feathers.output.MultipleSequenceFiles',
'text': 'fm.last.feathers.output.MultipleTextFiles',
'raw': 'fm.last.feathers.output.MultipleRawFileOutputFormat',
'sequencefile': 'fm.last.feathers.output.MultipleSequenceFiles'
}
else:
outputformat_shortcuts = {
'code': 'org.apache.hadoop.mapred.SequenceFileOutputFormat',
'text': 'org.apache.hadoop.mapred.TextOutputFormat',
'raw': 'fm.last.feathers.output.RawFileOutputFormat',
'sequencefile': 'org.apache.hadoop.mapred.SequenceFileOutputFormat'
}
outputformat_shortcuts.update(configopts('outputformats', self.prog))
outputformat = addedopts['outputformat'][0]
if outputformat.lower() in outputformat_shortcuts:
outputformat = outputformat_shortcuts[outputformat.lower()]
opts.add('outputformat', outputformat)
if addedopts['addpath'] and 'no' not in addedopts['addpath']:
opts.add('cmdenv', 'dumbo_addpath=true')
pyenv = envdef('PYTHONPATH', addedopts['libegg'], 'file', self.opts,
shortcuts=dict(configopts('eggs', self.prog)), quote=False, trim=True,
extrapaths=addedopts['pypath'])
if pyenv:
opts.add('cmdenv', pyenv)
hadenv = envdef('HADOOP_CLASSPATH', addedopts['libjar'], 'libjar',
self.opts, shortcuts=dict(configopts('jars', self.prog)))
tmpfiles = []
for _file in opts.pop('file'):
if _file.startswith('file://'):
opts.add('file', _file[7:])
else:
tmpfiles.append(_file)
if tmpfiles:
opts.add('jobconf', 'tmpfiles=%s' % ','.join(tmpfiles))
tmpjars = []
for jar in opts.pop('libjar'):
if jar.startswith('file://'):
opts.add('file', jar[7:])
else:
tmpjars.append(jar)
if tmpjars:
opts.add('jobconf', 'tmpjars=%s' % ','.join(tmpjars))
cmd = hadoop + '/bin/hadoop jar ' + streamingjar
retval = execute(cmd, opts, hadenv)
if 'yes' in addedopts['delinputs']:
inputs = opts['input']
for path in inputs:
execute("%s/bin/hadoop fs -rmr '%s'" % (hadoop, path))
return retval
def run(self):
retval = Iteration.run(self)
if retval != 0:
return retval
if os.path.exists(self.prog):
self.opts.append(("file", self.prog))
addedopts = getopts(
self.opts,
[
"hadoop",
"name",
"delinputs",
"libegg",
"libjar",
"inputformat",
"outputformat",
"nummaptasks",
"numreducetasks",
"priority",
"queue",
"cachefile",
"cachearchive",
"file",
"codewritable",
"addpath",
"getpath",
"python",
"streamoutput",
"pypath",
],
)
hadoop = findhadoop(addedopts["hadoop"][0])
streamingjar = findjar(hadoop, "streaming")
if not streamingjar:
print >> sys.stderr, "ERROR: Streaming jar not found"
return 1
try:
import typedbytes
except ImportError:
print >> sys.stderr, 'ERROR: "typedbytes" module not found'
return 1
modpath = re.sub("\.egg.*$", ".egg", typedbytes.__file__)
if modpath.endswith(".egg"):
addedopts["libegg"].append(modpath)
else:
self.opts.append(("file", modpath))
self.opts.append(("jobconf", "stream.map.input=typedbytes"))
self.opts.append(("jobconf", "stream.reduce.input=typedbytes"))
if addedopts["numreducetasks"] and addedopts["numreducetasks"][0] == "0":
self.opts.append(("jobconf", "stream.reduce.output=typedbytes"))
if addedopts["streamoutput"]:
id_ = addedopts["streamoutput"][0]
self.opts.append(("jobconf", "stream.map.output=" + id_))
else:
self.opts.append(("jobconf", "stream.map.output=typedbytes"))
else:
self.opts.append(("jobconf", "stream.map.output=typedbytes"))
if addedopts["streamoutput"]:
id_ = addedopts["streamoutput"][0]
self.opts.append(("jobconf", "stream.reduce.output=" + id_))
else:
self.opts.append(("jobconf", "stream.reduce.output=typedbytes"))
if not addedopts["name"]:
self.opts.append(("jobconf", "mapred.job.name=" + self.prog.split("/")[-1]))
else:
self.opts.append(("jobconf", "mapred.job.name=%s" % addedopts["name"][0]))
if addedopts["nummaptasks"]:
self.opts.append(("jobconf", "mapred.map.tasks=%s" % addedopts["nummaptasks"][0]))
if addedopts["numreducetasks"]:
numreducetasks = int(addedopts["numreducetasks"][0])
self.opts.append(("numReduceTasks", str(numreducetasks)))
if addedopts["priority"]:
self.opts.append(("jobconf", "mapred.job.priority=%s" % addedopts["priority"][0]))
if addedopts["queue"]:
self.opts.append(("jobconf", "mapred.job.queue.name=%s" % addedopts["queue"][0]))
if addedopts["cachefile"]:
for cachefile in addedopts["cachefile"]:
self.opts.append(("cacheFile", cachefile))
if addedopts["cachearchive"]:
for cachearchive in addedopts["cachearchive"]:
self.opts.append(("cacheArchive", cachearchive))
if addedopts["file"]:
for file in addedopts["file"]:
if not "://" in file:
if not os.path.exists(file):
raise ValueError('file "' + file + '" does not exist')
file = "file://" + os.path.abspath(file)
self.opts.append(("file", file))
if not addedopts["inputformat"]:
addedopts["inputformat"] = ["auto"]
inputformat_shortcuts = {
"code": "org.apache.hadoop.streaming.AutoInputFormat",
"text": "org.apache.hadoop.mapred.TextInputFormat",
"sequencefile": "org.apache.hadoop.streaming.AutoInputFormat",
"auto": "org.apache.hadoop.streaming.AutoInputFormat",
}
inputformat_shortcuts.update(configopts("inputformats", self.prog))
inputformat = addedopts["inputformat"][0]
if inputformat_shortcuts.has_key(inputformat.lower()):
inputformat = inputformat_shortcuts[inputformat.lower()]
self.opts.append(("inputformat", inputformat))
if not addedopts["outputformat"]:
addedopts["outputformat"] = ["sequencefile"]
if addedopts["getpath"] and addedopts["getpath"] != "no":
outputformat_shortcuts = {
"code": "fm.last.feathers.output.MultipleSequenceFiles",
"text": "fm.last.feathers.output.MultipleTextFiles",
"raw": "fm.last.feathers.output.MultipleRawFileOutputFormat",
"sequencefile": "fm.last.feathers.output.MultipleSequenceFiles",
}
else:
outputformat_shortcuts = {
"code": "org.apache.hadoop.mapred.SequenceFileOutputFormat",
"text": "org.apache.hadoop.mapred.TextOutputFormat",
"raw": "fm.last.feathers.output.RawFileOutputFormat",
"sequencefile": "org.apache.hadoop.mapred.SequenceFileOutputFormat",
}
outputformat_shortcuts.update(configopts("outputformats", self.prog))
outputformat = addedopts["outputformat"][0]
if outputformat_shortcuts.has_key(outputformat.lower()):
outputformat = outputformat_shortcuts[outputformat.lower()]
self.opts.append(("outputformat", outputformat))
if addedopts["addpath"] and addedopts["addpath"][0] != "no":
self.opts.append(("cmdenv", "dumbo_addpath=true"))
pyenv = envdef(
"PYTHONPATH",
addedopts["libegg"],
"file",
self.opts,
shortcuts=dict(configopts("eggs", self.prog)),
quote=False,
trim=True,
extrapaths=addedopts["pypath"],
)
if pyenv:
self.opts.append(("cmdenv", pyenv))
hadenv = envdef(
"HADOOP_CLASSPATH", addedopts["libjar"], "libjar", self.opts, shortcuts=dict(configopts("jars", self.prog))
)
fileopt = getopt(self.opts, "file")
if fileopt:
tmpfiles = []
for file in fileopt:
if file.startswith("file://"):
self.opts.append(("file", file[7:]))
else:
tmpfiles.append(file)
if tmpfiles:
self.opts.append(("jobconf", "tmpfiles=" + ",".join(tmpfiles)))
libjaropt = getopt(self.opts, "libjar")
if libjaropt:
tmpjars = []
for jar in libjaropt:
if jar.startswith("file://"):
self.opts.append(("file", jar[7:]))
else:
tmpjars.append(jar)
if tmpjars:
self.opts.append(("jobconf", "tmpjars=" + ",".join(tmpjars)))
cmd = hadoop + "/bin/hadoop jar " + streamingjar
retval = execute(cmd, self.opts, hadenv)
if addedopts["delinputs"] and addedopts["delinputs"][0] == "yes":
for (key, value) in self.opts:
if key == "input":
if os.path.exists(hadoop + "/bin/hdfs"):
hdfs = hadoop + "/bin/hdfs"
else:
hdfs = hadoop + "/bin/hadoop"
execute("%s dfs -rmr '%s'" % (hdfs, value))
return retval
def __init__(self, prog, opts):
Iteration.__init__(self, prog, opts)
self.opts += configopts('streaming', prog, self.opts)
hadoop = getopt(self.opts, 'hadoop', delete=False)[0]
self.opts += configopts('streaming_' + hadoop, prog, self.opts)