def train(self):
"""Start solver, we'll context switch to the caffe_root directory because Caffe has issues not being
the center of the universe.
"""
self.training = True
with cd(self.config.get('mri-client', 'caffe_root')):
caffe_path = self.config.get('mri-client', 'caffe_bin')
process_args = [
caffe_path, 'train', '--solver', self.directive['local_solver']
]
# Resume training from snapshot?
if 'resume' in self.directive:
process_args.append('--snapshot')
process_args.append(self.directive['resume'])
try:
proc = subprocess.Popen(process_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for raw_line in iter(proc.stderr.readline, b''):
decoded_line = raw_line.decode('utf-8')
line = decoded_line.replace('\n', '')
logging.debug('[CAFFE OUTPUT ] {0}'.format(line))
parsed_event = parse_caffe_train_line(line)
if parsed_event:
# If we've already seen the iteration, start sending data
if 'iteration' in parsed_event:
self.curiter = parsed_event['iteration']
if self.curiter is not None:
parsed_event['iteration'] = self.curiter
try:
event_struct = TrainingEvent(
parsed_event, 'iteration')
self.action_handler.put(event_struct)
except ValueError:
# We only want to send events that have a field filled out
pass
# Wait for completion
proc.wait()
logging.debug(
'Process finished...waiting extra for process thread to complete'
)
time.sleep(2)
code = proc.returncode
self.training = False
logging.debug('Caffe thread is now marked as finished')
except Exception as e:
raise e
finally:
pass
if code != 0:
logging.error(
'Caffe returned with non-zero error code! (returned {0})'.
format(code))
raise OSError('Caffe external call failed')
def train(self):
"""Start solver, we'll context switch to the caffe_root directory because Caffe has issues not being
the center of the universe.
"""
self.training = True
with cd(self.config.get('mri-client', 'caffe_root')):
caffe_path = self.config.get('mri-client', 'caffe_bin')
process_args = [caffe_path, 'train', '--solver', self.directive['local_solver']]
# Resume training from snapshot?
if 'resume' in self.directive:
process_args.append('--snapshot')
process_args.append(self.directive['resume'])
try:
proc = subprocess.Popen(process_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for raw_line in iter(proc.stderr.readline, b''):
decoded_line = raw_line.decode('utf-8')
line = decoded_line.replace('\n', '')
logging.debug('[CAFFE OUTPUT ] {0}'.format(line))
parsed_event = parse_caffe_train_line(line)
if parsed_event:
# If we've already seen the iteration, start sending data
if 'iteration' in parsed_event:
self.curiter = parsed_event['iteration']
if self.curiter is not None:
parsed_event['iteration'] = self.curiter
try:
event_struct = TrainingEvent(parsed_event, 'iteration')
self.action_handler.put(event_struct)
except ValueError:
# We only want to send events that have a field filled out
pass
# Wait for completion
proc.wait()
logging.debug('Process finished...waiting extra for process thread to complete')
time.sleep(2)
code = proc.returncode
self.training = False
logging.debug('Caffe thread is now marked as finished')
except Exception as e:
raise e
finally:
pass
if code != 0:
logging.error('Caffe returned with non-zero error code! (returned {0})'.format(code))
raise OSError('Caffe external call failed')
def test_cd(self):
tempdir = tempfile.gettempdir()
with cd(tempdir):
cwd = os.getcwd()
self.assertEqual(cwd, tempfile.tempdir)