def can_accumulate_gradients(self):
if config_value('caffe')['flavor'] == 'BVLC':
return True
elif config_value('caffe')['flavor'] == 'NVIDIA':
return (parse_version(config_value('caffe')['version']) > parse_version('0.14.0-alpha'))
else:
raise ValueError('Unknown flavor. Support NVIDIA and BVLC flavors only.')
def can_accumulate_gradients(self):
if config_value('caffe')['flavor'] == 'BVLC':
return True
elif config_value('caffe')['flavor'] == 'NVIDIA':
return (parse_version(config_value('caffe')['version']) > parse_version('0.14.0-alpha'))
else:
raise ValueError('Unknown flavor. Support NVIDIA and BVLC flavors only.')
def get_version_and_flavor(executable):
"""
Returns (version, flavor)
Should be called after import_pycaffe()
"""
version_string = get_version_from_pycaffe()
if version_string is None:
version_string = get_version_from_cmdline(executable)
if version_string is None:
version_string = get_version_from_soname(executable)
if version_string is None:
raise ValueError('Could not find version information for Caffe build ' +
'at "%s". Upgrade your installation' % executable)
version = parse_version(version_string)
if parse_version(0,99,0) > version > parse_version(0,9,0):
flavor = 'NVIDIA'
minimum_version = '0.11.0'
if version < parse_version(minimum_version):
raise ValueError(
'Required version "%s" is greater than "%s". Upgrade your installation.'
% (nvidia_minimum_version, version_string))
else:
flavor = 'BVLC'
return version_string, flavor
def get_executable_version_string(executable):
"""
Returns the caffe version as either a string from results of command line option '-version'
or None if '-version' not implemented
Arguments:
executable -- path to a caffe executable
"""
supported_platforms = ["Windows", "Linux", "Darwin"]
version_string = None
if platform.system() in supported_platforms:
p = subprocess.Popen([executable, "-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.wait():
raise config_option.BadValue(p.stderr.read().strip())
else:
pattern = "version"
for line in p.stdout:
if pattern in line:
version_string = line[line.find(pattern) + len(pattern) + 1 :].rstrip()
break
try:
parse_version(version_string)
except ValueError: # version_string is either ill-formatted or 'CAFFE_VERSION'
version_string = None
return version_string
else:
raise UnsupportedPlatformError('platform "%s" not supported' % platform.system())
def get_version_and_flavor(executable):
"""
Returns (version, flavor)
Should be called after import_pycaffe()
"""
version_string = get_version_from_pycaffe()
if version_string is None:
version_string = get_version_from_cmdline(executable)
if version_string is None:
version_string = get_version_from_soname(executable)
if version_string is None:
raise ValueError(
'Could not find version information for Caffe build ' +
'at "%s". Upgrade your installation' % executable)
version = parse_version(version_string)
if parse_version(0, 99, 0) > version > parse_version(0, 9, 0):
flavor = 'NVIDIA'
minimum_version = '0.11.0'
if version < parse_version(minimum_version):
raise ValueError(
'Required version "%s" is greater than "%s". Upgrade your installation.'
% (minimum_version, version_string))
else:
flavor = 'BVLC'
return version_string, flavor
def get_executable_version_string(executable):
"""
Returns the caffe version as either a string from results of command line option '-version'
or None if '-version' not implemented
Arguments:
executable -- path to a caffe executable
"""
supported_platforms = ['Windows', 'Linux', 'Darwin']
version_string = None
if platform.system() in supported_platforms:
p = subprocess.Popen([executable, '-version'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if p.wait():
raise config_option.BadValue(p.stderr.read().strip())
else:
pattern = 'version'
for line in p.stdout:
if pattern in line:
version_string = line[line.find(pattern) +
len(pattern) + 1:].rstrip()
break
try:
parse_version(version_string)
except ValueError: #version_string is either ill-formatted or 'CAFFE_VERSION'
version_string = None
return version_string
else:
raise UnsupportedPlatformError('platform "%s" not supported' %
platform.system())
def _set_config_dict_value(self, value):
if not value:
self._config_dict_value = None
else:
if value == '<PATHS>':
executable = self.find_executable('caffe')
if not executable:
executable = self.find_executable('caffe.exe')
else:
executable = os.path.join(value, 'build', 'tools', 'caffe')
info_dict = self.get_info(executable)
version = parse_version(info_dict['ver_str'])
if version >= parse_version(0, 12):
multi_gpu = True
else:
multi_gpu = False
flavor = info_dict['flavor']
# TODO: ask caffe for this information
cuda_enabled = len(device_query.get_devices()) > 0
self._config_dict_value = {
'executable': executable,
'version': version,
'ver_str': info_dict['ver_str'],
'multi_gpu': multi_gpu,
'cuda_enabled': cuda_enabled,
'flavor': flavor
}
def _set_config_dict_value(self, value):
if not value:
self._config_dict_value = None
else:
if value == '<PATHS>':
executable = self.find_executable('caffe')
if not executable:
executable = self.find_executable('caffe.exe')
else:
executable = os.path.join(value, 'build', 'tools', 'caffe')
info_dict = self.get_info(executable)
version = parse_version(info_dict['ver_str'])
if version >= parse_version(0,12):
multi_gpu = True
else:
multi_gpu = False
flavor = info_dict['flavor']
# TODO: ask caffe for this information
cuda_enabled = len(device_query.get_devices()) > 0
self._config_dict_value = {
'executable': executable,
'version': version,
'ver_str': info_dict['ver_str'],
'multi_gpu': multi_gpu,
'cuda_enabled': cuda_enabled,
'flavor': flavor
}
def _set_config_dict_value(self, value):
if not value:
self._config_dict_value = None
else:
if value == "<PATHS>":
executable = self.find_executable("caffe")
if not executable:
executable = self.find_executable("caffe.exe")
else:
executable = os.path.join(value, "build", "tools", "caffe")
info_dict = self.get_info(executable)
version = parse_version(info_dict["ver_str"])
if version >= parse_version(0, 12):
multi_gpu = True
else:
multi_gpu = False
flavor = info_dict["flavor"]
# TODO: ask caffe for this information
cuda_enabled = len(device_query.get_devices()) > 0
self._config_dict_value = {
"executable": executable,
"version": version,
"ver_str": info_dict["ver_str"],
"multi_gpu": multi_gpu,
"cuda_enabled": cuda_enabled,
"flavor": flavor,
}
def get_version(executable):
"""
Returns the caffe version as a (MAJOR, MINOR, PATCH) tuple or None
Arguments:
executable -- path to a caffe executable
"""
# TODO: check `caffe --version` when it's implemented
NVIDIA_SUFFIX = '-nv'
if platform.system() == 'Linux':
p = subprocess.Popen(['ldd', executable],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if p.wait():
raise config_option.BadValue(p.stderr.read().strip())
else:
libname = 'libcaffe'
caffe_line = None
# Search output for caffe library
for line in p.stdout:
if libname in line:
caffe_line = line
break
if caffe_line is None:
raise config_option.BadValue('%s not found in ldd output' %
libname)
# Read the symlink for libcaffe from ldd output
symlink = caffe_line.split()[2]
filename = os.path.basename(os.path.realpath(symlink))
# Check for the nvidia suffix
if NVIDIA_SUFFIX not in filename:
raise config_option.BadValue(
'Library at "%s" does not have expected suffix "%s". Are you using the NVIDIA/caffe fork?'
% (filename, NVIDIA_SUFFIX))
# parse the version string
match = re.match(
r'%s%s\.so\.(\S+)$' % (libname, NVIDIA_SUFFIX), filename)
if match:
version_str = match.group(1)
return parse_version(version_str)
else:
return None
elif platform.system() == 'Darwin':
# XXX: guess and let the user figure out errors later
return parse_version(0, 11, 0)
elif platform.system() == 'Windows':
# XXX: guess and let the user figure out errors later
return parse_version(0, 11, 0)
else:
print 'WARNING: platform "%s" not supported' % platform.system()
return None
def get_version(executable):
"""
Returns the caffe version as a (MAJOR, MINOR, PATCH) tuple or None
Arguments:
executable -- path to a caffe executable
"""
# TODO: check `caffe --version` when it's implemented
NVIDIA_SUFFIX = '-nv'
if platform.system() == 'Linux':
p = subprocess.Popen(['ldd', executable],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
if p.wait():
raise config_option.BadValue(p.stderr.read().strip())
else:
libname = 'libcaffe'
caffe_line = None
# Search output for caffe library
for line in p.stdout:
if libname in line:
caffe_line = line
break
if caffe_line is None:
raise config_option.BadValue('%s not found in ldd output' % libname)
# Read the symlink for libcaffe from ldd output
symlink = caffe_line.split()[2]
filename = os.path.basename(os.path.realpath(symlink))
# Check for the nvidia suffix
if NVIDIA_SUFFIX not in filename:
raise config_option.BadValue('Library at "%s" does not have expected suffix "%s". Are you using the NVIDIA/caffe fork?'
% (filename, NVIDIA_SUFFIX))
# parse the version string
match = re.match(r'%s%s\.so\.(\S+)$'
% (libname, NVIDIA_SUFFIX), filename)
if match:
version_str = match.group(1)
return parse_version(version_str)
else:
return None
elif platform.system() == 'Darwin':
# XXX: guess and let the user figure out errors later
return parse_version(0,11,0)
elif platform.system() == 'Windows':
# XXX: guess and let the user figure out errors later
return parse_version(0,11,0)
else:
print 'WARNING: platform "%s" not supported' % platform.system()
return None
def get_flavor(ver_str):
"""
Returns the information about caffe library enhancement (NVIDIA or BVLC)
Arguments:
ver_str -- version string that can identify enhancement flavor
"""
if parse_version(0, 99, 0) > parse_version(ver_str) > parse_version(0, 9, 0):
return "NVIDIA"
else:
return "BVLC"
class DistributedCaffeFramework(CaffeFramework):
"""
derive from CaffeFramework, use for training in long-distance server
"""
"""
Defines required methods to interact with the Caffe framework
This class can be instantiated as many times as there are compatible
instances of Caffe
"""
# short descriptive name
NAME = 'Caffe'
# identifier of framework class (intended to be the same across
# all instances of this class)
CLASS = 'caffe'
# whether this framework can shuffle data during training
CAN_SHUFFLE_DATA = False
SUPPORTS_PYTHON_LAYERS_FILE = True
SUPPORTS_TIMELINE_TRACING = False
if config_value('caffe')['flavor'] == 'NVIDIA':
if parse_version(config_value('caffe')['version']) > parse_version(
'0.14.0-alpha'):
SUPPORTED_SOLVER_TYPES = [
'SGD', 'NESTEROV', 'ADAGRAD', 'RMSPROP', 'ADADELTA', 'ADAM'
]
else:
SUPPORTED_SOLVER_TYPES = ['SGD', 'NESTEROV', 'ADAGRAD']
elif config_value('caffe')['flavor'] == 'BVLC':
SUPPORTED_SOLVER_TYPES = [
'SGD', 'NESTEROV', 'ADAGRAD', 'RMSPROP', 'ADADELTA', 'ADAM'
]
else:
raise ValueError(
'Unknown flavor. Support NVIDIA and BVLC flavors only.')
SUPPORTED_DATA_TRANSFORMATION_TYPES = ['MEAN_SUBTRACTION', 'CROPPING']
SUPPORTED_DATA_AUGMENTATION_TYPES = []
@override
def __init__(self):
super(CaffeFramework, self).__init__()
self.framework_id = self.CLASS
@override
def create_train_task(self, **kwargs):
"""
create train task
"""
print 'return DistributedTrainTask'
return DistributedTrainTask(framework_id=self.framework_id, **kwargs)
def get_flavor(ver_str):
"""
Returns the information about caffe library enhancement (NVIDIA or BVLC)
Arguments:
ver_str -- version string that can identify enhancement flavor
"""
if parse_version(0, 99, 0) > parse_version(ver_str) > parse_version(
0, 9, 0):
return 'NVIDIA'
else:
return 'BVLC'
def _set_config_dict_value(self, value):
if not value:
self._config_dict_value = None
else:
if value == '<PATHS>':
executable = self.find_executable('caffe')
if not executable:
executable = self.find_executable('caffe.exe')
else:
executable = 'T:/Caffe/Build/x64/Release/caffe.exe'#os.path.join(value, 'build', 'tools', 'caffe')
version = self.get_version(executable)
if version >= parse_version(0,12):
multi_gpu = True
else:
multi_gpu = False
# TODO: ask caffe for this information
cuda_enabled = len(device_query.get_devices()) > 0
self._config_dict_value = {
'executable': executable,
'version': version,
'multi_gpu': multi_gpu,
'cuda_enabled': cuda_enabled,
}
def validate_version(cls, executable):
"""
Utility for checking the caffe version from within validate()
Throws BadValue
Arguments:
executable -- path to a caffe executable
"""
nvidia_minimum_version = '0.11.0'
info_dict = cls.get_info(executable)
if info_dict['ver_str'] is None:
raise config_option.BadValue('Your Caffe does not have version info. Please upgrade it.')
else:
flavor = CaffeOption.get_flavor(info_dict['ver_str'])
if flavor == 'NVIDIA' and parse_version(nvidia_minimum_version) > parse_version(info_dict['ver_str']):
raise config_option.BadValue(
'Required version "{min_ver}" is greater than "{running_ver}". '\
'Upgrade your installation.'\
.format(min_ver = nvidia_minimum_version, running_ver = info_dict['ver_str']))
else:
return True
def validate_version(cls, executable):
"""
Utility for checking the caffe version from within validate()
Throws BadValue
Arguments:
executable -- path to a caffe executable
"""
nvidia_minimum_version = '0.11.0'
info_dict = cls.get_info(executable)
if info_dict['ver_str'] is None:
raise config_option.BadValue(
'Your Caffe does not have version info. Please upgrade it.')
else:
flavor = CaffeOption.get_flavor(info_dict['ver_str'])
if flavor == 'NVIDIA' and parse_version(
nvidia_minimum_version) > parse_version(
info_dict['ver_str']):
raise config_option.BadValue(
'Required version "{min_ver}" is greater than "{running_ver}". '\
'Upgrade your installation.'\
.format(min_ver = nvidia_minimum_version, running_ver = info_dict['ver_str']))
else:
return True
def validate_version(cls, executable):
"""
Utility for checking the caffe version from within validate()
Throws BadValue
Arguments:
executable -- path to a caffe executable
"""
minimum_version = parse_version(0,11,0)
version = cls.get_version(executable)
if version is None:
raise config_option.BadValue('Could not get version information from caffe at "%s". Are you using the NVIDIA fork?'
% executable)
elif minimum_version > version:
raise config_option.BadValue('Required version "%s" is greater than "%s". Upgrade your installation.'
% (
'.'.join(str(n) for n in minimum_version),
'.'.join(str(n) for n in version)
))
else:
return True
def can_accumulate_gradients(self):
return (config_value('caffe_root')['version'] >
parse_version('0.14.0-alpha'))
# Read the symlink for libcaffe from ldd output
symlink = caffe_line.split()[2]
filename = os.path.basename(os.path.realpath(symlink))
# parse the version string
match = re.match(r'%s(.*)\.so\.(\S+)$' % (libname), filename)
if match:
return match.group(2)
else:
return None
if 'CAFFE_ROOT' in os.environ:
executable, version, flavor = load_from_envvar('CAFFE_ROOT')
elif 'CAFFE_HOME' in os.environ:
executable, version, flavor = load_from_envvar('CAFFE_HOME')
else:
executable, version, flavor = load_from_path()
option_list['caffe'] = {
'executable':
executable,
'version':
version,
'flavor':
flavor,
'multi_gpu': (flavor == 'BVLC'
or parse_version(version) >= parse_version(0, 12)),
'cuda_enabled': (len(device_query.get_devices()) > 0),
}
if caffe_line is None:
raise ValueError('libcaffe not found in linked libraries for "%s"'
% executable)
# Read the symlink for libcaffe from ldd output
symlink = caffe_line.split()[2]
filename = os.path.basename(os.path.realpath(symlink))
# parse the version string
match = re.match(r'%s(.*)\.so\.(\S+)$' % (libname), filename)
if match:
return match.group(2)
else:
return None
if 'CAFFE_ROOT' in os.environ:
executable, version, flavor = load_from_envvar('CAFFE_ROOT')
elif 'CAFFE_HOME' in os.environ:
executable, version, flavor = load_from_envvar('CAFFE_HOME')
else:
executable, version, flavor = load_from_path()
option_list['caffe'] = {
'executable': executable,
'version': version,
'flavor': flavor,
'multi_gpu': (flavor == 'BVLC' or parse_version(version) >= parse_version(0, 12)),
'cuda_enabled': (len(device_query.get_devices()) > 0),
}
class CaffeFramework(Framework):
"""
Defines required methods to interact with the Caffe framework
This class can be instantiated as many times as there are compatible
instances of Caffe
"""
# short descriptive name
NAME = 'Caffe'
# identifier of framework class (intended to be the same across
# all instances of this class)
CLASS = 'caffe'
# whether this framework can shuffle data during training
CAN_SHUFFLE_DATA = False
SUPPORTS_PYTHON_LAYERS_FILE = True
SUPPORTS_TIMELINE_TRACING = False
if config_value('caffe')['flavor'] == 'NVIDIA':
if parse_version(config_value('caffe')['version']) > parse_version(
'0.14.0-alpha'):
SUPPORTED_SOLVER_TYPES = [
'SGD', 'NESTEROV', 'ADAGRAD', 'RMSPROP', 'ADADELTA', 'ADAM'
]
else:
SUPPORTED_SOLVER_TYPES = ['SGD', 'NESTEROV', 'ADAGRAD']
elif config_value('caffe')['flavor'] == 'BVLC':
SUPPORTED_SOLVER_TYPES = [
'SGD', 'NESTEROV', 'ADAGRAD', 'RMSPROP', 'ADADELTA', 'ADAM'
]
else:
raise ValueError(
'Unknown flavor. Support NVIDIA and BVLC flavors only.')
SUPPORTED_DATA_TRANSFORMATION_TYPES = ['MEAN_SUBTRACTION', 'CROPPING']
SUPPORTED_DATA_AUGMENTATION_TYPES = []
@override
def __init__(self):
super(CaffeFramework, self).__init__()
self.framework_id = self.CLASS
@override
def create_train_task(self, **kwargs):
"""
create train task
"""
return CaffeTrainTask(framework_id=self.framework_id, **kwargs)
@override
def validate_network(self, data):
"""
validate a network (input data are expected to be a text
description of the network)
"""
pb = caffe_pb2.NetParameter()
try:
text_format.Merge(data, pb)
except text_format.ParseError as e:
raise BadNetworkError('Not a valid NetParameter: %s' % e)
@override
def get_standard_network_desc(self, network):
"""
return description of standard network
network is expected to be a instance of caffe_pb2.NetParameter
"""
networks_dir = os.path.join(os.path.dirname(digits.__file__),
'standard-networks', self.CLASS)
for filename in os.listdir(networks_dir):
path = os.path.join(networks_dir, filename)
if os.path.isfile(path):
match = None
match = re.match(r'%s.prototxt' % network, filename)
if match:
with open(path) as infile:
return infile.read()
# return None if not found
return None
@override
def get_network_from_desc(self, network_desc):
"""
return network object from a string representation
"""
network = caffe_pb2.NetParameter()
text_format.Merge(network_desc, network)
return network
@override
def get_network_from_previous(self, previous_network, use_same_dataset):
"""
return new instance of network from previous network
"""
network = caffe_pb2.NetParameter()
network.CopyFrom(previous_network)
if not use_same_dataset:
# Rename the final layer
# XXX making some assumptions about network architecture here
ip_layers = [l for l in network.layer if l.type == 'InnerProduct']
if len(ip_layers) > 0:
ip_layers[-1].name = '%s_retrain' % ip_layers[-1].name
return network
@override
def get_network_from_path(self, path):
"""
return network object from a file path
"""
network = caffe_pb2.NetParameter()
with open(path) as infile:
text_format.Merge(infile.read(), network)
return network
@override
def get_network_visualization(self, **kwargs):
"""
return visualization of network
"""
desc = kwargs['desc']
net = caffe_pb2.NetParameter()
text_format.Merge(desc, net)
# Throws an error if name is None
if not net.name:
net.name = 'Network'
return ('<image src="data:image/png;base64,' +
caffe.draw.draw_net(net, 'UD').encode('base64') +
'" style="max-width:100%" />')
@override
def can_accumulate_gradients(self):
if config_value('caffe')['flavor'] == 'BVLC':
return True
elif config_value('caffe')['flavor'] == 'NVIDIA':
return (parse_version(config_value('caffe')['version']) >
parse_version('0.14.0-alpha'))
else:
raise ValueError(
'Unknown flavor. Support NVIDIA and BVLC flavors only.')
def can_accumulate_gradients(self):
return (config_value('caffe_root')['version']
> parse_version('0.14.0-alpha'))
% executable)
# Read the symlink for libcaffe from ldd output
symlink = caffe_line.split()[2]
filename = os.path.basename(os.path.realpath(symlink))
# parse the version string
match = re.match(r'%s(.*)\.so\.(\S+)$' % (libname), filename)
if match:
return match.group(2)
else:
return None
if 'CAFFE_ROOT' in os.environ:
executable, version, flavor = load_from_envvar('CAFFE_ROOT')
elif 'CAFFE_HOME' in os.environ:
executable, version, flavor = load_from_envvar('CAFFE_HOME')
else:
executable, version, flavor = load_from_path()
option_list['caffe'] = {
'executable': executable,
'version': version,
'flavor': flavor,
'multi_gpu': (flavor == 'BVLC' or parse_version(version) >= parse_version(0,12)),
'cuda_enabled': (len(device_query.get_devices()) > 0),
}