def Run(self, main_file):
cmd = main_file
for key, value in self.parameter:
if "file" not in key:
value = value.lower()
cmd += " --f2m_%s=%s" % (key, value)
CmdTool.run_cmd_and_wait(CmdTool(), cmd)
def __init__(self, options, rank, sock):
CmdTool.__init__(self, options.mapreduce_ssh_port)
self.options = options
self.rank = rank
self.process = None
self.sock = sock
if self.is_map_worker():
type = 'Mapper'
local_rank = rank
else:
type = 'Reducer'
local_rank = rank - options.num_map_worker
task = options.all_tasks[rank]
self.name = '%s-%s(%s, %s)' % (type, local_rank, task['machine'],
task['class'])
def __init__(self, options, rank, sock):
CmdTool.__init__(self, options.mapreduce_ssh_port)
self.options = options
self.rank = rank
self.process = None
self.sock = sock
if self.is_map_worker():
type = 'Mapper'
local_rank = rank
else:
type = 'Reducer'
local_rank = rank - options.num_map_worker
task = options.all_tasks[rank]
self.name = '%s-%s(%s, %s)' %(type,
local_rank,
task['machine'],
task['class'])
def __init__(self, options):
""" Copy agent, server and user-specified client
to specified directory on remote machines
"""
CmdTool.__init__(self, options['ssh_port'])
self.all_socks = []
self.process_list = []
self.options = options
""" Clear files
"""
logging.info('Clear files at %s' %time.asctime())
self.clear_files(self.options['working_directory'],
self.options['machine_list'])
""" dispatch files
"""
logging.info('Start dispatch files to remote nodes at %s' %time.asctime())
file_list = [options['client_name'],
options['brook'],
options['machine_file']]
dir_dict = {}
for machine in options['machine_list']:
dir_dict[machine[0]] = options['working_directory']
self.dispatch_file(file_list, dir_dict)
logging.info('End dispatch files at %s' %time.asctime())
""" Create FIFO file
"""
logging.info('Start create FIFO file at %s' %time.asctime())
FIFO_list = [options['parameter_signal'],
options['statistics_signal'],
options['counter_file']]
self.create_file(FIFO_list, options)
logging.info('End create FIFO file at %s' %time.asctime())
def __init__(self, options):
""" Copy agent, server and user-specified client
to specified directory on remote machines
"""
CmdTool.__init__(self, options['ssh_port'])
self.all_socks = []
self.process_list = []
self.options = options
""" Clear files
"""
logging.info('Clear files at %s' % time.asctime())
self.clear_files(self.options['working_directory'],
self.options['machine_list'])
""" dispatch files
"""
logging.info('Start dispatch files to remote nodes at %s' %
time.asctime())
file_list = [
options['client_name'], options['brook'], options['machine_file']
]
dir_dict = {}
for machine in options['machine_list']:
dir_dict[machine[0]] = options['working_directory']
self.dispatch_file(file_list, dir_dict)
logging.info('End dispatch files at %s' % time.asctime())
""" Create FIFO file
"""
logging.info('Start create FIFO file at %s' % time.asctime())
FIFO_list = [
options['parameter_signal'], options['statistics_signal'],
options['counter_file']
]
self.create_file(FIFO_list, options)
logging.info('End create FIFO file at %s' % time.asctime())
def __init__(self, options):
""" Copy python script worker.py and user-specified executable
to specified directory on remote machines
"""
CmdTool.__init__(self, options.mapreduce_ssh_port)
self.options = options
self.all_socks = []
self.reduce_socks = None
# copy map-reduce executable and worker script to each machine
cmd = 'cp %s %s/%s' % (options.local_executable, options.script_path,
options.remote_executable)
self.run_cmd_and_wait(cmd)
executable = '%s/%s' % (options.script_path, options.remote_executable)
file_list = [executable]
for script in ALL_SCRIPTS:
local_script = '%s/%s' % (options.script_path, script)
file_list.append(local_script)
mesg = 'Copy binary executable and python scripts to machines'
logging.info(mesg)
self.dispatch_file(file_list, options.mapreduce_tmp_dir)
self.run_cmd_and_wait('rm -rf %s' % executable)
def __init__(self, options):
""" Copy python script worker.py and user-specified executable
to specified directory on remote machines
"""
CmdTool.__init__(self, options.mapreduce_ssh_port)
self.options= options
self.all_socks = []
self.reduce_socks = None
# copy map-reduce executable and worker script to each machine
cmd = 'cp %s %s/%s' %(options.local_executable,
options.script_path,
options.remote_executable)
self.run_cmd_and_wait(cmd)
executable = '%s/%s' %(options.script_path, options.remote_executable)
file_list = [executable]
for script in ALL_SCRIPTS:
local_script = '%s/%s' %(options.script_path, script)
file_list.append(local_script)
mesg = 'Copy binary executable and python scripts to machines'
logging.info(mesg)
self.dispatch_file(file_list, options.mapreduce_tmp_dir)
self.run_cmd_and_wait('rm -rf %s' %executable)