예제 #1
0
 def Run(self, main_file):
     cmd = main_file
     for key, value in self.parameter:
         if "file" not in key:
             value = value.lower()
         cmd += " --f2m_%s=%s" % (key, value)
     CmdTool.run_cmd_and_wait(CmdTool(), cmd)
예제 #2
0
 def __init__(self, options, rank, sock):
     CmdTool.__init__(self, options.mapreduce_ssh_port)
     self.options = options
     self.rank = rank
     self.process = None
     self.sock = sock
     if self.is_map_worker():
         type = 'Mapper'
         local_rank = rank
     else:
         type = 'Reducer'
         local_rank = rank - options.num_map_worker
     task = options.all_tasks[rank]
     self.name = '%s-%s(%s, %s)' % (type, local_rank, task['machine'],
                                    task['class'])
예제 #3
0
 def __init__(self, options, rank, sock):
     CmdTool.__init__(self, options.mapreduce_ssh_port)
     self.options = options
     self.rank = rank
     self.process = None
     self.sock = sock
     if self.is_map_worker():
         type = 'Mapper'
         local_rank = rank 
     else:
         type = 'Reducer'
         local_rank = rank - options.num_map_worker
     task = options.all_tasks[rank]
     self.name = '%s-%s(%s, %s)' %(type,
                                   local_rank,
                                   task['machine'],
                                   task['class'])
예제 #4
0
    def __init__(self, options):
        """ Copy agent, server and user-specified client 
        to specified directory on remote machines
        """
        CmdTool.__init__(self, options['ssh_port'])
        self.all_socks = []
        self.process_list = []
        self.options = options

        """ Clear files
        """
        logging.info('Clear files at %s' %time.asctime())
        self.clear_files(self.options['working_directory'], 
                         self.options['machine_list']) 

        """ dispatch files
        """
        logging.info('Start dispatch files to remote nodes at %s' %time.asctime())

        file_list = [options['client_name'], 
                     options['brook'],
                     options['machine_file']]
        dir_dict = {}

        for machine in options['machine_list']:
            dir_dict[machine[0]] = options['working_directory']

        self.dispatch_file(file_list, dir_dict)
        logging.info('End dispatch files at %s' %time.asctime())

        """ Create FIFO file
        """
        logging.info('Start create FIFO file at %s' %time.asctime())

        FIFO_list = [options['parameter_signal'],
                     options['statistics_signal'],
                     options['counter_file']]

        self.create_file(FIFO_list, options)

        logging.info('End create FIFO file at %s' %time.asctime())
예제 #5
0
    def __init__(self, options):
        """ Copy agent, server and user-specified client 
        to specified directory on remote machines
        """
        CmdTool.__init__(self, options['ssh_port'])
        self.all_socks = []
        self.process_list = []
        self.options = options
        """ Clear files
        """
        logging.info('Clear files at %s' % time.asctime())
        self.clear_files(self.options['working_directory'],
                         self.options['machine_list'])
        """ dispatch files
        """
        logging.info('Start dispatch files to remote nodes at %s' %
                     time.asctime())

        file_list = [
            options['client_name'], options['brook'], options['machine_file']
        ]
        dir_dict = {}

        for machine in options['machine_list']:
            dir_dict[machine[0]] = options['working_directory']

        self.dispatch_file(file_list, dir_dict)
        logging.info('End dispatch files at %s' % time.asctime())
        """ Create FIFO file
        """
        logging.info('Start create FIFO file at %s' % time.asctime())

        FIFO_list = [
            options['parameter_signal'], options['statistics_signal'],
            options['counter_file']
        ]

        self.create_file(FIFO_list, options)

        logging.info('End create FIFO file at %s' % time.asctime())
예제 #6
0
    def __init__(self, options):
        """ Copy python script worker.py and user-specified executable
        to specified directory on remote machines
        """
        CmdTool.__init__(self, options.mapreduce_ssh_port)
        self.options = options
        self.all_socks = []
        self.reduce_socks = None

        # copy map-reduce executable and worker script to each machine
        cmd = 'cp %s %s/%s' % (options.local_executable, options.script_path,
                               options.remote_executable)
        self.run_cmd_and_wait(cmd)
        executable = '%s/%s' % (options.script_path, options.remote_executable)
        file_list = [executable]
        for script in ALL_SCRIPTS:
            local_script = '%s/%s' % (options.script_path, script)
            file_list.append(local_script)
        mesg = 'Copy binary executable and python scripts to machines'
        logging.info(mesg)
        self.dispatch_file(file_list, options.mapreduce_tmp_dir)
        self.run_cmd_and_wait('rm -rf %s' % executable)
예제 #7
0
    def __init__(self, options):
        """ Copy python script worker.py and user-specified executable
        to specified directory on remote machines
        """
        CmdTool.__init__(self, options.mapreduce_ssh_port)
        self.options= options
        self.all_socks = []
        self.reduce_socks = None

        # copy map-reduce executable and worker script to each machine
        cmd = 'cp %s %s/%s' %(options.local_executable, 
                              options.script_path,
                              options.remote_executable)
        self.run_cmd_and_wait(cmd)
        executable = '%s/%s' %(options.script_path, options.remote_executable)
        file_list = [executable]
        for script in ALL_SCRIPTS:
            local_script = '%s/%s' %(options.script_path, script)
            file_list.append(local_script)
        mesg = 'Copy binary executable and python scripts to machines'
	logging.info(mesg)
        self.dispatch_file(file_list, options.mapreduce_tmp_dir)
        self.run_cmd_and_wait('rm -rf %s' %executable)