Ejemplo n.º 1
0
    def train(self):
        """Start solver, we'll context switch to the caffe_root directory because Caffe has issues not being
        the center of the universe.
        """
        self.training = True
        with cd(self.config.get('mri-client', 'caffe_root')):
            caffe_path = self.config.get('mri-client', 'caffe_bin')
            process_args = [
                caffe_path, 'train', '--solver', self.directive['local_solver']
            ]
            # Resume training from snapshot?
            if 'resume' in self.directive:
                process_args.append('--snapshot')
                process_args.append(self.directive['resume'])
            try:
                proc = subprocess.Popen(process_args,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
                for raw_line in iter(proc.stderr.readline, b''):
                    decoded_line = raw_line.decode('utf-8')
                    line = decoded_line.replace('\n', '')
                    logging.debug('[CAFFE OUTPUT ] {0}'.format(line))
                    parsed_event = parse_caffe_train_line(line)
                    if parsed_event:
                        # If we've already seen the iteration, start sending data
                        if 'iteration' in parsed_event:
                            self.curiter = parsed_event['iteration']
                        if self.curiter is not None:
                            parsed_event['iteration'] = self.curiter
                            try:
                                event_struct = TrainingEvent(
                                    parsed_event, 'iteration')
                                self.action_handler.put(event_struct)
                            except ValueError:
                                # We only want to send events that have a field filled out
                                pass

                # Wait for completion
                proc.wait()
                logging.debug(
                    'Process finished...waiting extra for process thread to complete'
                )
                time.sleep(2)
                code = proc.returncode
                self.training = False
                logging.debug('Caffe thread is now marked as finished')
            except Exception as e:
                raise e
            finally:
                pass
        if code != 0:
            logging.error(
                'Caffe returned with non-zero error code! (returned {0})'.
                format(code))
            raise OSError('Caffe external call failed')
Ejemplo n.º 2
0
    def train(self):
        """Start solver, we'll context switch to the caffe_root directory because Caffe has issues not being
        the center of the universe.
        """
        self.training = True
        with cd(self.config.get('mri-client', 'caffe_root')):
            caffe_path = self.config.get('mri-client', 'caffe_bin')
            process_args = [caffe_path, 'train', '--solver', self.directive['local_solver']]
            # Resume training from snapshot?
            if 'resume' in self.directive:
                process_args.append('--snapshot')
                process_args.append(self.directive['resume'])
            try:
                proc = subprocess.Popen(process_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                for raw_line in iter(proc.stderr.readline, b''):
                    decoded_line = raw_line.decode('utf-8')
                    line = decoded_line.replace('\n', '')
                    logging.debug('[CAFFE OUTPUT ] {0}'.format(line))
                    parsed_event = parse_caffe_train_line(line)
                    if parsed_event:
                        # If we've already seen the iteration, start sending data
                        if 'iteration' in parsed_event:
                            self.curiter = parsed_event['iteration']
                        if self.curiter is not None:
                            parsed_event['iteration'] = self.curiter
                            try:
                                event_struct = TrainingEvent(parsed_event, 'iteration')
                                self.action_handler.put(event_struct)
                            except ValueError:
                                # We only want to send events that have a field filled out
                                pass

                # Wait for completion
                proc.wait()
                logging.debug('Process finished...waiting extra for process thread to complete')
                time.sleep(2)
                code = proc.returncode
                self.training = False
                logging.debug('Caffe thread is now marked as finished')
            except Exception as e:
                raise e
            finally:
                pass
        if code != 0:
            logging.error('Caffe returned with non-zero error code! (returned {0})'.format(code))
            raise OSError('Caffe external call failed')
Ejemplo n.º 3
0
 def test_cd(self):
     tempdir = tempfile.gettempdir()
     with cd(tempdir):
         cwd = os.getcwd()
         self.assertEqual(cwd, tempfile.tempdir)