def can_accumulate_gradients(self): if config_value('caffe')['flavor'] == 'BVLC': return True elif config_value('caffe')['flavor'] == 'NVIDIA': return (parse_version(config_value('caffe')['version']) > parse_version('0.14.0-alpha')) else: raise ValueError('Unknown flavor. Support NVIDIA and BVLC flavors only.')
def get_version_and_flavor(executable): """ Returns (version, flavor) Should be called after import_pycaffe() """ version_string = get_version_from_pycaffe() if version_string is None: version_string = get_version_from_cmdline(executable) if version_string is None: version_string = get_version_from_soname(executable) if version_string is None: raise ValueError('Could not find version information for Caffe build ' + 'at "%s". Upgrade your installation' % executable) version = parse_version(version_string) if parse_version(0,99,0) > version > parse_version(0,9,0): flavor = 'NVIDIA' minimum_version = '0.11.0' if version < parse_version(minimum_version): raise ValueError( 'Required version "%s" is greater than "%s". Upgrade your installation.' % (nvidia_minimum_version, version_string)) else: flavor = 'BVLC' return version_string, flavor
def get_executable_version_string(executable): """ Returns the caffe version as either a string from results of command line option '-version' or None if '-version' not implemented Arguments: executable -- path to a caffe executable """ supported_platforms = ["Windows", "Linux", "Darwin"] version_string = None if platform.system() in supported_platforms: p = subprocess.Popen([executable, "-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.wait(): raise config_option.BadValue(p.stderr.read().strip()) else: pattern = "version" for line in p.stdout: if pattern in line: version_string = line[line.find(pattern) + len(pattern) + 1 :].rstrip() break try: parse_version(version_string) except ValueError: # version_string is either ill-formatted or 'CAFFE_VERSION' version_string = None return version_string else: raise UnsupportedPlatformError('platform "%s" not supported' % platform.system())
def get_version_and_flavor(executable): """ Returns (version, flavor) Should be called after import_pycaffe() """ version_string = get_version_from_pycaffe() if version_string is None: version_string = get_version_from_cmdline(executable) if version_string is None: version_string = get_version_from_soname(executable) if version_string is None: raise ValueError( 'Could not find version information for Caffe build ' + 'at "%s". Upgrade your installation' % executable) version = parse_version(version_string) if parse_version(0, 99, 0) > version > parse_version(0, 9, 0): flavor = 'NVIDIA' minimum_version = '0.11.0' if version < parse_version(minimum_version): raise ValueError( 'Required version "%s" is greater than "%s". Upgrade your installation.' % (minimum_version, version_string)) else: flavor = 'BVLC' return version_string, flavor
def get_executable_version_string(executable): """ Returns the caffe version as either a string from results of command line option '-version' or None if '-version' not implemented Arguments: executable -- path to a caffe executable """ supported_platforms = ['Windows', 'Linux', 'Darwin'] version_string = None if platform.system() in supported_platforms: p = subprocess.Popen([executable, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.wait(): raise config_option.BadValue(p.stderr.read().strip()) else: pattern = 'version' for line in p.stdout: if pattern in line: version_string = line[line.find(pattern) + len(pattern) + 1:].rstrip() break try: parse_version(version_string) except ValueError: #version_string is either ill-formatted or 'CAFFE_VERSION' version_string = None return version_string else: raise UnsupportedPlatformError('platform "%s" not supported' % platform.system())
def _set_config_dict_value(self, value): if not value: self._config_dict_value = None else: if value == '<PATHS>': executable = self.find_executable('caffe') if not executable: executable = self.find_executable('caffe.exe') else: executable = os.path.join(value, 'build', 'tools', 'caffe') info_dict = self.get_info(executable) version = parse_version(info_dict['ver_str']) if version >= parse_version(0, 12): multi_gpu = True else: multi_gpu = False flavor = info_dict['flavor'] # TODO: ask caffe for this information cuda_enabled = len(device_query.get_devices()) > 0 self._config_dict_value = { 'executable': executable, 'version': version, 'ver_str': info_dict['ver_str'], 'multi_gpu': multi_gpu, 'cuda_enabled': cuda_enabled, 'flavor': flavor }
def _set_config_dict_value(self, value): if not value: self._config_dict_value = None else: if value == '<PATHS>': executable = self.find_executable('caffe') if not executable: executable = self.find_executable('caffe.exe') else: executable = os.path.join(value, 'build', 'tools', 'caffe') info_dict = self.get_info(executable) version = parse_version(info_dict['ver_str']) if version >= parse_version(0,12): multi_gpu = True else: multi_gpu = False flavor = info_dict['flavor'] # TODO: ask caffe for this information cuda_enabled = len(device_query.get_devices()) > 0 self._config_dict_value = { 'executable': executable, 'version': version, 'ver_str': info_dict['ver_str'], 'multi_gpu': multi_gpu, 'cuda_enabled': cuda_enabled, 'flavor': flavor }
def _set_config_dict_value(self, value): if not value: self._config_dict_value = None else: if value == "<PATHS>": executable = self.find_executable("caffe") if not executable: executable = self.find_executable("caffe.exe") else: executable = os.path.join(value, "build", "tools", "caffe") info_dict = self.get_info(executable) version = parse_version(info_dict["ver_str"]) if version >= parse_version(0, 12): multi_gpu = True else: multi_gpu = False flavor = info_dict["flavor"] # TODO: ask caffe for this information cuda_enabled = len(device_query.get_devices()) > 0 self._config_dict_value = { "executable": executable, "version": version, "ver_str": info_dict["ver_str"], "multi_gpu": multi_gpu, "cuda_enabled": cuda_enabled, "flavor": flavor, }
def get_version(executable): """ Returns the caffe version as a (MAJOR, MINOR, PATCH) tuple or None Arguments: executable -- path to a caffe executable """ # TODO: check `caffe --version` when it's implemented NVIDIA_SUFFIX = '-nv' if platform.system() == 'Linux': p = subprocess.Popen(['ldd', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.wait(): raise config_option.BadValue(p.stderr.read().strip()) else: libname = 'libcaffe' caffe_line = None # Search output for caffe library for line in p.stdout: if libname in line: caffe_line = line break if caffe_line is None: raise config_option.BadValue('%s not found in ldd output' % libname) # Read the symlink for libcaffe from ldd output symlink = caffe_line.split()[2] filename = os.path.basename(os.path.realpath(symlink)) # Check for the nvidia suffix if NVIDIA_SUFFIX not in filename: raise config_option.BadValue( 'Library at "%s" does not have expected suffix "%s". Are you using the NVIDIA/caffe fork?' % (filename, NVIDIA_SUFFIX)) # parse the version string match = re.match( r'%s%s\.so\.(\S+)$' % (libname, NVIDIA_SUFFIX), filename) if match: version_str = match.group(1) return parse_version(version_str) else: return None elif platform.system() == 'Darwin': # XXX: guess and let the user figure out errors later return parse_version(0, 11, 0) elif platform.system() == 'Windows': # XXX: guess and let the user figure out errors later return parse_version(0, 11, 0) else: print 'WARNING: platform "%s" not supported' % platform.system() return None
def get_version(executable): """ Returns the caffe version as a (MAJOR, MINOR, PATCH) tuple or None Arguments: executable -- path to a caffe executable """ # TODO: check `caffe --version` when it's implemented NVIDIA_SUFFIX = '-nv' if platform.system() == 'Linux': p = subprocess.Popen(['ldd', executable], stdout = subprocess.PIPE, stderr = subprocess.PIPE) if p.wait(): raise config_option.BadValue(p.stderr.read().strip()) else: libname = 'libcaffe' caffe_line = None # Search output for caffe library for line in p.stdout: if libname in line: caffe_line = line break if caffe_line is None: raise config_option.BadValue('%s not found in ldd output' % libname) # Read the symlink for libcaffe from ldd output symlink = caffe_line.split()[2] filename = os.path.basename(os.path.realpath(symlink)) # Check for the nvidia suffix if NVIDIA_SUFFIX not in filename: raise config_option.BadValue('Library at "%s" does not have expected suffix "%s". Are you using the NVIDIA/caffe fork?' % (filename, NVIDIA_SUFFIX)) # parse the version string match = re.match(r'%s%s\.so\.(\S+)$' % (libname, NVIDIA_SUFFIX), filename) if match: version_str = match.group(1) return parse_version(version_str) else: return None elif platform.system() == 'Darwin': # XXX: guess and let the user figure out errors later return parse_version(0,11,0) elif platform.system() == 'Windows': # XXX: guess and let the user figure out errors later return parse_version(0,11,0) else: print 'WARNING: platform "%s" not supported' % platform.system() return None
def get_flavor(ver_str): """ Returns the information about caffe library enhancement (NVIDIA or BVLC) Arguments: ver_str -- version string that can identify enhancement flavor """ if parse_version(0, 99, 0) > parse_version(ver_str) > parse_version(0, 9, 0): return "NVIDIA" else: return "BVLC"
class DistributedCaffeFramework(CaffeFramework): """ derive from CaffeFramework, use for training in long-distance server """ """ Defines required methods to interact with the Caffe framework This class can be instantiated as many times as there are compatible instances of Caffe """ # short descriptive name NAME = 'Caffe' # identifier of framework class (intended to be the same across # all instances of this class) CLASS = 'caffe' # whether this framework can shuffle data during training CAN_SHUFFLE_DATA = False SUPPORTS_PYTHON_LAYERS_FILE = True SUPPORTS_TIMELINE_TRACING = False if config_value('caffe')['flavor'] == 'NVIDIA': if parse_version(config_value('caffe')['version']) > parse_version( '0.14.0-alpha'): SUPPORTED_SOLVER_TYPES = [ 'SGD', 'NESTEROV', 'ADAGRAD', 'RMSPROP', 'ADADELTA', 'ADAM' ] else: SUPPORTED_SOLVER_TYPES = ['SGD', 'NESTEROV', 'ADAGRAD'] elif config_value('caffe')['flavor'] == 'BVLC': SUPPORTED_SOLVER_TYPES = [ 'SGD', 'NESTEROV', 'ADAGRAD', 'RMSPROP', 'ADADELTA', 'ADAM' ] else: raise ValueError( 'Unknown flavor. Support NVIDIA and BVLC flavors only.') SUPPORTED_DATA_TRANSFORMATION_TYPES = ['MEAN_SUBTRACTION', 'CROPPING'] SUPPORTED_DATA_AUGMENTATION_TYPES = [] @override def __init__(self): super(CaffeFramework, self).__init__() self.framework_id = self.CLASS @override def create_train_task(self, **kwargs): """ create train task """ print 'return DistributedTrainTask' return DistributedTrainTask(framework_id=self.framework_id, **kwargs)
def get_flavor(ver_str): """ Returns the information about caffe library enhancement (NVIDIA or BVLC) Arguments: ver_str -- version string that can identify enhancement flavor """ if parse_version(0, 99, 0) > parse_version(ver_str) > parse_version( 0, 9, 0): return 'NVIDIA' else: return 'BVLC'
def _set_config_dict_value(self, value): if not value: self._config_dict_value = None else: if value == '<PATHS>': executable = self.find_executable('caffe') if not executable: executable = self.find_executable('caffe.exe') else: executable = 'T:/Caffe/Build/x64/Release/caffe.exe'#os.path.join(value, 'build', 'tools', 'caffe') version = self.get_version(executable) if version >= parse_version(0,12): multi_gpu = True else: multi_gpu = False # TODO: ask caffe for this information cuda_enabled = len(device_query.get_devices()) > 0 self._config_dict_value = { 'executable': executable, 'version': version, 'multi_gpu': multi_gpu, 'cuda_enabled': cuda_enabled, }
def validate_version(cls, executable): """ Utility for checking the caffe version from within validate() Throws BadValue Arguments: executable -- path to a caffe executable """ nvidia_minimum_version = '0.11.0' info_dict = cls.get_info(executable) if info_dict['ver_str'] is None: raise config_option.BadValue('Your Caffe does not have version info. Please upgrade it.') else: flavor = CaffeOption.get_flavor(info_dict['ver_str']) if flavor == 'NVIDIA' and parse_version(nvidia_minimum_version) > parse_version(info_dict['ver_str']): raise config_option.BadValue( 'Required version "{min_ver}" is greater than "{running_ver}". '\ 'Upgrade your installation.'\ .format(min_ver = nvidia_minimum_version, running_ver = info_dict['ver_str'])) else: return True
def validate_version(cls, executable): """ Utility for checking the caffe version from within validate() Throws BadValue Arguments: executable -- path to a caffe executable """ nvidia_minimum_version = '0.11.0' info_dict = cls.get_info(executable) if info_dict['ver_str'] is None: raise config_option.BadValue( 'Your Caffe does not have version info. Please upgrade it.') else: flavor = CaffeOption.get_flavor(info_dict['ver_str']) if flavor == 'NVIDIA' and parse_version( nvidia_minimum_version) > parse_version( info_dict['ver_str']): raise config_option.BadValue( 'Required version "{min_ver}" is greater than "{running_ver}". '\ 'Upgrade your installation.'\ .format(min_ver = nvidia_minimum_version, running_ver = info_dict['ver_str'])) else: return True
def validate_version(cls, executable): """ Utility for checking the caffe version from within validate() Throws BadValue Arguments: executable -- path to a caffe executable """ minimum_version = parse_version(0,11,0) version = cls.get_version(executable) if version is None: raise config_option.BadValue('Could not get version information from caffe at "%s". Are you using the NVIDIA fork?' % executable) elif minimum_version > version: raise config_option.BadValue('Required version "%s" is greater than "%s". Upgrade your installation.' % ( '.'.join(str(n) for n in minimum_version), '.'.join(str(n) for n in version) )) else: return True
def can_accumulate_gradients(self): return (config_value('caffe_root')['version'] > parse_version('0.14.0-alpha'))
# Read the symlink for libcaffe from ldd output symlink = caffe_line.split()[2] filename = os.path.basename(os.path.realpath(symlink)) # parse the version string match = re.match(r'%s(.*)\.so\.(\S+)$' % (libname), filename) if match: return match.group(2) else: return None if 'CAFFE_ROOT' in os.environ: executable, version, flavor = load_from_envvar('CAFFE_ROOT') elif 'CAFFE_HOME' in os.environ: executable, version, flavor = load_from_envvar('CAFFE_HOME') else: executable, version, flavor = load_from_path() option_list['caffe'] = { 'executable': executable, 'version': version, 'flavor': flavor, 'multi_gpu': (flavor == 'BVLC' or parse_version(version) >= parse_version(0, 12)), 'cuda_enabled': (len(device_query.get_devices()) > 0), }
if caffe_line is None: raise ValueError('libcaffe not found in linked libraries for "%s"' % executable) # Read the symlink for libcaffe from ldd output symlink = caffe_line.split()[2] filename = os.path.basename(os.path.realpath(symlink)) # parse the version string match = re.match(r'%s(.*)\.so\.(\S+)$' % (libname), filename) if match: return match.group(2) else: return None if 'CAFFE_ROOT' in os.environ: executable, version, flavor = load_from_envvar('CAFFE_ROOT') elif 'CAFFE_HOME' in os.environ: executable, version, flavor = load_from_envvar('CAFFE_HOME') else: executable, version, flavor = load_from_path() option_list['caffe'] = { 'executable': executable, 'version': version, 'flavor': flavor, 'multi_gpu': (flavor == 'BVLC' or parse_version(version) >= parse_version(0, 12)), 'cuda_enabled': (len(device_query.get_devices()) > 0), }
class CaffeFramework(Framework): """ Defines required methods to interact with the Caffe framework This class can be instantiated as many times as there are compatible instances of Caffe """ # short descriptive name NAME = 'Caffe' # identifier of framework class (intended to be the same across # all instances of this class) CLASS = 'caffe' # whether this framework can shuffle data during training CAN_SHUFFLE_DATA = False SUPPORTS_PYTHON_LAYERS_FILE = True SUPPORTS_TIMELINE_TRACING = False if config_value('caffe')['flavor'] == 'NVIDIA': if parse_version(config_value('caffe')['version']) > parse_version( '0.14.0-alpha'): SUPPORTED_SOLVER_TYPES = [ 'SGD', 'NESTEROV', 'ADAGRAD', 'RMSPROP', 'ADADELTA', 'ADAM' ] else: SUPPORTED_SOLVER_TYPES = ['SGD', 'NESTEROV', 'ADAGRAD'] elif config_value('caffe')['flavor'] == 'BVLC': SUPPORTED_SOLVER_TYPES = [ 'SGD', 'NESTEROV', 'ADAGRAD', 'RMSPROP', 'ADADELTA', 'ADAM' ] else: raise ValueError( 'Unknown flavor. Support NVIDIA and BVLC flavors only.') SUPPORTED_DATA_TRANSFORMATION_TYPES = ['MEAN_SUBTRACTION', 'CROPPING'] SUPPORTED_DATA_AUGMENTATION_TYPES = [] @override def __init__(self): super(CaffeFramework, self).__init__() self.framework_id = self.CLASS @override def create_train_task(self, **kwargs): """ create train task """ return CaffeTrainTask(framework_id=self.framework_id, **kwargs) @override def validate_network(self, data): """ validate a network (input data are expected to be a text description of the network) """ pb = caffe_pb2.NetParameter() try: text_format.Merge(data, pb) except text_format.ParseError as e: raise BadNetworkError('Not a valid NetParameter: %s' % e) @override def get_standard_network_desc(self, network): """ return description of standard network network is expected to be a instance of caffe_pb2.NetParameter """ networks_dir = os.path.join(os.path.dirname(digits.__file__), 'standard-networks', self.CLASS) for filename in os.listdir(networks_dir): path = os.path.join(networks_dir, filename) if os.path.isfile(path): match = None match = re.match(r'%s.prototxt' % network, filename) if match: with open(path) as infile: return infile.read() # return None if not found return None @override def get_network_from_desc(self, network_desc): """ return network object from a string representation """ network = caffe_pb2.NetParameter() text_format.Merge(network_desc, network) return network @override def get_network_from_previous(self, previous_network, use_same_dataset): """ return new instance of network from previous network """ network = caffe_pb2.NetParameter() network.CopyFrom(previous_network) if not use_same_dataset: # Rename the final layer # XXX making some assumptions about network architecture here ip_layers = [l for l in network.layer if l.type == 'InnerProduct'] if len(ip_layers) > 0: ip_layers[-1].name = '%s_retrain' % ip_layers[-1].name return network @override def get_network_from_path(self, path): """ return network object from a file path """ network = caffe_pb2.NetParameter() with open(path) as infile: text_format.Merge(infile.read(), network) return network @override def get_network_visualization(self, **kwargs): """ return visualization of network """ desc = kwargs['desc'] net = caffe_pb2.NetParameter() text_format.Merge(desc, net) # Throws an error if name is None if not net.name: net.name = 'Network' return ('<image src="data:image/png;base64,' + caffe.draw.draw_net(net, 'UD').encode('base64') + '" style="max-width:100%" />') @override def can_accumulate_gradients(self): if config_value('caffe')['flavor'] == 'BVLC': return True elif config_value('caffe')['flavor'] == 'NVIDIA': return (parse_version(config_value('caffe')['version']) > parse_version('0.14.0-alpha')) else: raise ValueError( 'Unknown flavor. Support NVIDIA and BVLC flavors only.')
% executable) # Read the symlink for libcaffe from ldd output symlink = caffe_line.split()[2] filename = os.path.basename(os.path.realpath(symlink)) # parse the version string match = re.match(r'%s(.*)\.so\.(\S+)$' % (libname), filename) if match: return match.group(2) else: return None if 'CAFFE_ROOT' in os.environ: executable, version, flavor = load_from_envvar('CAFFE_ROOT') elif 'CAFFE_HOME' in os.environ: executable, version, flavor = load_from_envvar('CAFFE_HOME') else: executable, version, flavor = load_from_path() option_list['caffe'] = { 'executable': executable, 'version': version, 'flavor': flavor, 'multi_gpu': (flavor == 'BVLC' or parse_version(version) >= parse_version(0,12)), 'cuda_enabled': (len(device_query.get_devices()) > 0), }