def __setstate__(self, state):
super(ImageClassificationDatasetJob, self).__setstate__(state)
if self.pickver_job_dataset_image_classification <= 1:
print 'Upgrading ImageClassificationDatasetJob to version 2'
task = self.train_db_task()
if task.image_dims[2] == 3:
if task.encoding == "jpg":
if task.mean_file.endswith('.binaryproto'):
print '\tConverting mean file "%s" from RGB to BGR.' % task.path(
task.mean_file)
import numpy as np
import caffe_pb2
old_blob = caffe_pb2.BlobProto()
with open(task.path(task.mean_file), 'rb') as infile:
old_blob.ParseFromString(infile.read())
data = np.array(old_blob.data).reshape(
old_blob.channels, old_blob.height, old_blob.width)
data = data[[2, 1, 0], ...] # channel swap
new_blob = caffe_pb2.BlobProto()
new_blob.num = 1
new_blob.channels, new_blob.height, new_blob.width = data.shape
new_blob.data.extend(data.astype(float).flat)
with open(task.path(task.mean_file), 'wb') as outfile:
outfile.write(new_blob.SerializeToString())
else:
print '\tSetting "%s" status to ERROR because it was created with RGB channels' % self.name(
)
self.status = Status.ERROR
for task in self.tasks:
task.status = Status.ERROR
task.exception = 'This dataset was created with unencoded RGB channels. Caffe requires BGR input.'
self.pickver_job_dataset_image_classification = PICKLE_VERSION
def create_mean_file(self):
filename = os.path.join(self.job_dir, constants.MEAN_FILE_IMAGE)
# don't recreate file if it already exists
if not os.path.exists(filename):
mean_file = self.dataset.get_mean_file()
assert mean_file != None and mean_file.endswith(
'.binaryproto'
), 'Mean subtraction required but dataset has no mean file in .binaryproto format'
blob = caffe_pb2.BlobProto()
with open(self.dataset.path(mean_file), 'rb') as infile:
blob.ParseFromString(infile.read())
data = np.array(blob.data,
dtype=np.uint8).reshape(blob.channels, blob.height,
blob.width)
if blob.channels == 3:
# converting from BGR to RGB
data = data[[2, 1, 0], ...] # channel swap
# convert to (height, width, channels)
data = data.transpose((1, 2, 0))
else:
assert blob.channels == 1
# convert to (height, width)
data = data[0]
# save to file
image = PIL.Image.fromarray(data)
image.save(filename)
return filename
def save_mean(self, feature_sum, entry_count, dataset_dir, stage):
"""
Save mean to file
"""
data = np.around(feature_sum / entry_count).astype(np.uint8)
mean_file = os.path.join(stage, 'mean.binaryproto')
# Transform to caffe's format requirements
if data.ndim == 3:
if data.shape[0] == 3:
# channel swap
# XXX see issue #59
data = data[[2, 1, 0], ...]
elif data.ndim == 2:
# Add a channels axis
data = data[np.newaxis, :, :]
blob = caffe_pb2.BlobProto()
blob.num = 1
blob.channels, blob.height, blob.width = data.shape
blob.data.extend(data.astype(float).flat)
with open(os.path.join(dataset_dir, mean_file), 'wb') as outfile:
outfile.write(blob.SerializeToString())
logger.info('Created mean file for stage %s in %s' %
(stage, mean_file))
def readMeanFile(mean_path=''):
with open(mean_path, 'rb') as f:
blob = caffe_proto.BlobProto()
blob.ParseFromString(f.read())
f.close()
return {
'data': blob.data._values,
# 'channels':blob.channels,
# 'width':blob.width,
# 'height':blob.height
}
def array_to_blobproto(arr, diff=None):
"""Converts a N-dimensional array to blob proto. If diff is given, also
convert the diff. You need to make sure that arr and diff have the same
shape, and this function does not do sanity check.
"""
blob = caffe_pb2.BlobProto()
blob.shape.dim.extend(arr.shape)
blob.data.extend(arr.astype(float).flat)
if diff is not None:
blob.diff.extend(diff.astype(float).flat)
return blob
def array_to_blobproto(arr, diff=None):
"""Converts a 4-dimensional array to blob proto. If diff is given, also
convert the diff. You need to make sure that arr and diff have the same
shape, and this function does not do sanity check.
"""
if arr.ndim != 4:
raise ValueError('Incorrect array shape.')
blob = caffe_pb2.BlobProto()
blob.num, blob.channels, blob.height, blob.width = arr.shape
blob.data.extend(arr.astype(float).flat)
if diff is not None:
blob.diff.extend(diff.astype(float).flat)
return blob
def convert_array_to_blob(arr):
"""convert numpy array to caffe blob
Parameters
==========
arr: np.array
array
Returns
=======
blob: caffe.BlobProto
blob
"""
blob = caffe_pb2.BlobProto()
blob.shape.dim.extend(arr.shape)
blob.data.extend(arr.flatten())
return blob
def show_alpha(iter):
blob_file = blob_file_tpl % iter
blob = caffe_pb2.BlobProto()
f = open(blob_file, 'rb')
blob.ParseFromString(f.read())
f.close()
print 'blob: %sx%sx%sx%s' % (blob.num, blob.channels, blob.height,
blob.width)
img = np.zeros((blob.height, blob.width), dtype=np.uint8)
for idx, p in enumerate(blob.data):
value = min(max((p + 1.0) * 128, 0), 255)
img[idx / blob.width, idx % blob.width] = np.uint8(value)
# cv2.imshow("alpha", img)
cv2.imwrite(blob_file + ".bmp", img)
def get_transformer(self):
"""
Returns an instance of caffe.io.Transformer
"""
# check if already loaded
if hasattr(self, '_transformer') and self._transformer is not None:
return self._transformer
data_shape = (1, self.dataset.image_dims[2])
if self.crop_size:
data_shape += (self.crop_size, self.crop_size)
else:
data_shape += (self.dataset.image_dims[0],
self.dataset.image_dims[1])
t = caffe.io.Transformer(inputs={'data': data_shape})
t.set_transpose('data',
(2, 0, 1)) # transpose to (channels, height, width)
if self.dataset.image_dims[2] == 3 and \
self.dataset.train_db_task().image_channel_order == 'BGR':
# channel swap
# XXX see issue #59
t.set_channel_swap('data', (2, 1, 0))
if self.use_mean:
# set mean
with open(self.dataset.path(
self.dataset.train_db_task().mean_file)) as f:
blob = caffe_pb2.BlobProto()
blob.MergeFromString(f.read())
pixel = np.reshape(blob.data, (
self.dataset.image_dims[2],
self.dataset.image_dims[0],
self.dataset.image_dims[1],
)).mean(1).mean(1)
t.set_mean('data', pixel)
#t.set_raw_scale('data', 255) # [0,255] range instead of [0,1]
self._transformer = t
return self._transformer
def generate_submit_file(iter):
out_file = open(out_file_tpl % iter, 'w')
out_file.write(header)
f_idx = 0
line_idx = 0
while os.path.isfile(blob_file_tpl % (iter, f_idx)):
blob_file = blob_file_tpl % (iter, f_idx)
blob = caffe_pb2.BlobProto()
f = open(blob_file, 'rb')
blob.ParseFromString(f.read())
f.close()
print 'blob: %sx%sx%sx%s' % (blob.num, blob.channels, blob.height,
blob.width)
for i in range(blob.num):
print >> out_file, '%s,' % f_name[line_idx] + ','.join(
map(str, blob.data[i * 121:(i + 1) * 121]))
line_idx += 1
f_idx += 1
out_file.close()
def _save_mean(mean, filename):
"""
Saves mean to file
Arguments:
mean -- the mean as an np.ndarray
filename -- the location to save the image
"""
if filename.endswith('.binaryproto'):
blob = caffe_pb2.BlobProto()
blob.num = 1
blob.channels = mean.shape[0]
blob.height = mean.shape[1]
blob.width = mean.shape[2]
blob.data.extend(mean.astype(float).flat)
with open(filename, 'wb') as outfile:
outfile.write(blob.SerializeToString())
elif filename.endswith(('.jpg', '.jpeg', '.png')):
_save_image(mean, filename)
else:
raise ValueError('unrecognized file extension')
def _save_means(image_sum, image_count, mean_files):
"""
Save mean[s] to file
"""
mean = np.around(image_sum / image_count).astype(np.uint8)
for mean_file in mean_files:
if mean_file.lower().endswith('.npy'):
np.save(mean_file, mean)
elif mean_file.lower().endswith('.binaryproto'):
data = mean
# Transform to caffe's format requirements
if data.ndim == 3:
# Transpose to (channels, height, width)
data = data.transpose((2, 0, 1))
if data.shape[0] == 3:
# channel swap
# XXX see issue #59
data = data[[2, 1, 0], ...]
elif mean.ndim == 2:
# Add a channels axis
data = data[np.newaxis, :, :]
blob = caffe_pb2.BlobProto()
blob.num = 1
blob.channels, blob.height, blob.width = data.shape
blob.data.extend(data.astype(float).flat)
with open(mean_file, 'wb') as outfile:
outfile.write(blob.SerializeToString())
elif mean_file.lower().endswith(('.jpg', '.jpeg', '.png')):
image = PIL.Image.fromarray(mean)
image.save(mean_file)
else:
logger.warning('Unrecognized file extension for mean file: "%s"' %
mean_file)
continue
logger.info('Mean saved at "%s"' % mean_file)
help='Path to model.caffemodel')
parser.add_argument('--data',
help='Print all arrays in full',
action='store_true')
parser.add_argument(
'--codegenDir',
help=
'Path to an existing temporary directory to save generated protobuf Python classes',
default=tempfile.mkdtemp())
args = parser.parse_args()
local_caffe_proto = os.path.join(args.codegenDir,
os.path.basename(args.caffe_proto))
with open(local_caffe_proto, 'w') as f:
f.write((urllib2.urlopen(args.caffe_proto)
if 'http' in args.caffe_proto else open(args.caffe_proto)).read())
subprocess.check_call([
'protoc', '--proto_path',
os.path.dirname(local_caffe_proto), '--python_out', args.codegenDir,
local_caffe_proto
])
sys.path.insert(0, args.codegenDir)
import caffe_pb2
deserialized = caffe_pb2.NetParameter() if os.path.splitext(
args.model_caffemodel)[1] == '.caffemodel' else caffe_pb2.BlobProto()
deserialized.ParseFromString(open(args.model_caffemodel, 'rb').read())
json.dump(pb2json(deserialized, args.data), sys.stdout, indent=2)
def as_blob(array):
blob = pb2.BlobProto()
blob.shape.dim.extend(array.shape)
blob.data.extend(array.astype(float).flat)
return blob
def create(self, input_file, width, height,
channels = 3,
resize_mode = None,
image_folder= None,
shuffle = True,
mean_files = None,
encoding = 'none',
):
"""
Read an input file and create a database from the specified image/label pairs
Returns True on success
Arguments:
input_file -- gives paths to images and their label (e.g. "path/to/image1.jpg 0\npath/to/image2.jpg 3")
width -- width of resized images
height -- width of resized images
Keyword arguments:
channels -- channels of resized images
resize_mode -- can be crop, squash, fill or half_crop
image_folder -- folder in which the images can be found
shuffle -- shuffle images before saving
mean_files -- an array of mean files to save (can be empty)
encoding -- 'none', 'png' or 'jpg'
"""
### Validate input
if not os.path.exists(input_file):
logger.error('input_file does not exist')
return False
if height <= 0:
logger.error('unsupported image height')
return False
self.height = height
if width <= 0:
logger.error('unsupported image width')
return False
self.width = width
if channels not in [1,3]:
logger.error('unsupported number of channels')
return False
self.channels = channels
if resize_mode not in ['crop', 'squash', 'fill', 'half_crop']:
logger.error('unsupported resize_mode')
return False
self.resize_mode = resize_mode
if image_folder is not None and not os.path.exists(image_folder):
logger.error('image_folder does not exist')
return False
self.image_folder = image_folder
if mean_files:
for mean_file in mean_files:
if os.path.exists(mean_file):
logger.warning('overwriting existing mean file "%s"!' % mean_file)
else:
dirname = os.path.dirname(mean_file)
if not dirname:
dirname = '.'
if not os.path.exists(dirname):
logger.error('Cannot save mean file at "%s"' % mean_file)
return False
self.compute_mean = (mean_files and len(mean_files) > 0)
if encoding not in ['none', 'png', 'jpg']:
raise ValueError('Unsupported encoding format "%s"' % encoding)
self.encoding = encoding
### Start working
start = time.time()
# TODO: adjust these values in real-time based on system load
if not shuffle:
#XXX This is the only way to preserve order for now
# This obviously hurts performance considerably
read_threads = 1
write_threads = 1
else:
read_threads = 10
write_threads = 10
batch_size = 100
total_images_added = 0
total_image_sum = None
# NOTE: The data could really stack up in these queues
self.read_queue = Queue.Queue()
self.write_queue = Queue.Queue(2*batch_size)
# Tells read threads that if read_queue is empty, they can stop
self.read_queue_built = threading.Event()
self.read_thread_results = Queue.Queue()
# Tells write threads that if write_queue is empty, they can stop
self.write_queue_built = threading.Event()
self.write_thread_results = Queue.Queue()
# Read input_file and produce items to read_queue
# NOTE This secion should be very efficient, because no progress about the job gets reported until after the read/write threads start
lines_read = 0
lines_per_category = {}
with open(input_file, 'r') as f:
lines = f.readlines()
if shuffle:
random.shuffle(lines)
for line in lines:
# Expect format - [/]path/to/file.jpg 123
match = re_match(r'(.+)\s+(\d+)\s*$', line)
if match is not None:
path = match.group(1)
label = int(match.group(2))
self.read_queue.put( (path, label) )
if label not in lines_per_category:
lines_per_category[label] = 1
else:
lines_per_category[label] += 1
lines_read += 1
self.read_queue_built.set()
if lines_read > 0:
logger.info('Input images: %d' % lines_read)
else:
logger.error('no lines in input_file')
return False
for key in sorted(lines_per_category):
logger.debug('Category %s has %d images.' % (key, lines_per_category[key]))
# Start read threads
for i in xrange(read_threads):
p = threading.Thread(target=self.read_thread)
p.daemon = True
p.start()
# Start write threads
for i in xrange(write_threads):
first_batch = int(batch_size * (i+1)/write_threads)
p = threading.Thread(target=self.write_thread, args=(batch_size, first_batch))
p.daemon = True
p.start()
# Wait for threads to finish
wait_time = time.time()
read_threads_done = 0
write_threads_done = 0
total_images_written = 0
while write_threads_done < write_threads:
if self.shutdown.is_set():
# Die immediately
return False
# Send update every 2 seconds
if time.time() - wait_time > 2:
logger.debug('Processed %d/%d' % (lines_read - self.read_queue.qsize(), lines_read))
#print '\tRead queue size: %d' % self.read_queue.qsize()
#print '\tWrite queue size: %d' % self.write_queue.qsize()
#print '\tRead threads done: %d' % read_threads_done
#print '\tWrite threads done: %d' % write_threads_done
wait_time = time.time()
if not self.write_queue_built.is_set() and read_threads_done == read_threads:
self.write_queue_built.set()
while not self.read_thread_results.empty():
images_added, image_sum = self.read_thread_results.get()
total_images_added += images_added
# Update total_image_sum
if self.compute_mean and images_added > 0 and image_sum is not None:
if total_image_sum is None:
total_image_sum = image_sum
else:
total_image_sum += image_sum
read_threads_done += 1
while not self.write_thread_results.empty():
result = self.write_thread_results.get()
total_images_written += result
write_threads_done += 1
try:
time.sleep(0.2)
except KeyboardInterrupt:
self.shutdown.set()
return False
if total_images_added == 0:
logger.error('no images added')
return False
# Compute image mean
if self.compute_mean and total_image_sum is not None:
mean = np.around(total_image_sum / total_images_added).astype(np.uint8)
for mean_file in mean_files:
if mean_file.lower().endswith('.npy'):
np.save(mean_file, mean)
elif mean_file.lower().endswith('.binaryproto'):
data = mean
# Transform to caffe's format requirements
if data.ndim == 3:
# Transpose to (channels, height, width)
data = data.transpose((2,0,1))
if data.shape[0] == 3:
# channel swap
# XXX see issue #59
data = data[[2,1,0],...]
elif mean.ndim == 2:
# Add a channels axis
data = data[np.newaxis,:,:]
blob = caffe_pb2.BlobProto()
blob.num = 1
blob.channels, blob.height, blob.width = data.shape
blob.data.extend(data.astype(float).flat)
with open(mean_file, 'w') as outfile:
outfile.write(blob.SerializeToString())
elif mean_file.lower().endswith(('.jpg', '.jpeg', '.png')):
image = PIL.Image.fromarray(mean)
image.save(mean_file)
else:
logger.warning('Unrecognized file extension for mean file: "%s"' % mean_file)
continue
logger.info('Mean saved at "%s"' % mean_file)
logger.info('Database created after %d seconds.' % (time.time() - start))
logger.info('Total images added: %d' % total_images_written)
self.shutdown.set()
return True
# https://raw.githubusercontent.com/BVLC/caffe/master/src/caffe/proto/caffe.proto
codegenDir = os.path.dirname(os.path.abspath(__file__)) + '/tmp'
if not os.path.exists(codegenDir): os.makedirs(codegenDir)
local_caffe_proto = os.path.join(codegenDir, 'caffe.proto')
subprocess.check_call([
'protoc', '--proto_path',
os.path.dirname(local_caffe_proto), '--python_out', codegenDir,
local_caffe_proto
])
sys.path.insert(0, codegenDir)
import caffe_pb2
deserialized = caffe_pb2.NetParameter() if os.path.splitext(
args.mpath)[1] == '.caffemodel' else caffe_pb2.BlobProto()
deserialized.ParseFromString(open(args.mpath, 'rb').read())
# json.dump(pb2json(deserialized, args.data), sys.stdout, indent = 2)
jsobj = pb2json(deserialized, False)
with open('small.json', 'w') as jsfile:
json.dump(jsobj, jsfile, indent=2)
if weight is not None: npobj[npname + '.weight'] = weight
if bias is not None: npobj[npname + '.bias'] = bias
weight = None
bias = None
np.save('m', npobj)
# a = np.load('m.npy')
# m = a[()]
import csv
import base64
labels = list()
with open('./stanford-cars-model/labels.csv', 'rb') as csvfile:
labels_data = csv.reader(csvfile, delimiter=',')
for row in labels_data:
labels.append(row[1])
import caffe
import caffe_pb2
caffe.set_mode_cpu()
mean_blob = caffe_pb2.BlobProto()
with open('./stanford-cars-model/mean.binaryproto') as f:
mean_blob.ParseFromString(f.read())
mean_array = numpy.asarray(mean_blob.data, dtype=numpy.float32).reshape(
(mean_blob.channels, mean_blob.height, mean_blob.width))
model_def = './stanford-cars-model/deploy.prototxt'
model_weights = './stanford-cars-model/caffe_model_1_iter_22169.caffemodel'
net = caffe.Net(
model_def, # defines the structure of the model
model_weights, # contains the trained weights
caffe.TEST) # use test mode (e.g., don't perform dropout)
net.blobs['data'].reshape(
1, # batch size
def getBlob(weight):
blob = caffe_pb2.BlobProto()
blob.raw_data_type = caffe_pb2.FLOAT
blob.raw_data = weight.data.tobytes()
blob.shape.dim.extend(weight.shape)
return blob
with zipfile.ZipFile('weights.zip', 'w') as f:
for l in cq2.layer:
if hasattr(l, 'blobs'):
if len(l.blobs) > 0:
name = l.name
count = 0
for b in l.blobs:
data = np.array(b.data, dtype=np.float32)
shape = np.array(b.shape.dim, dtype=np.int32)
data = data.reshape(shape)
data = _to_npy_str(data)
save_name = name + ('-%d.npy' % count)
count += 1
f.writestr(save_name, data)
print(save_name)
# parse the mean file
# note that the channels for caffe model are BGR not RGB
# https://github.com/BVLC/caffe/wiki/Image-Format:-BGR-not-RGB
with open('places365CNN_mean.binaryproto', 'rb') as f:
cq3 = cq.BlobProto()
cq3.ParseFromString(f.read())
mean_img = np.array(cq3.data).reshape((cq3.channels, cq3.height, cq3.width))
np.save('places365CNN_mean.npy', mean_img)
def save_prototxt_files(self):
"""
Save solver, train_val and deploy files to disk
"""
has_val_set = self.dataset.val_db_task() is not None
### Check what has been specified in self.network
tops = []
bottoms = {}
train_data_layer = None
val_data_layer = None
hidden_layers = caffe_pb2.NetParameter()
loss_layers = []
accuracy_layers = []
for layer in self.network.layer:
assert layer.type not in ['MemoryData', 'HDF5Data', 'ImageData'
], 'unsupported data layer type'
if layer.type == 'Data':
for rule in layer.include:
if rule.phase == caffe_pb2.TRAIN:
assert train_data_layer is None, 'cannot specify two train data layers'
train_data_layer = layer
elif rule.phase == caffe_pb2.TEST:
assert val_data_layer is None, 'cannot specify two test data layers'
val_data_layer = layer
elif layer.type == 'SoftmaxWithLoss':
loss_layers.append(layer)
elif layer.type == 'Accuracy':
addThis = True
if layer.accuracy_param.HasField('top_k'):
if layer.accuracy_param.top_k >= len(self.get_labels()):
self.logger.warning(
'Removing layer %s because top_k=%s while there are are only %s labels in this dataset'
% (layer.name, layer.accuracy_param.top_k,
len(self.get_labels())))
addThis = False
if addThis:
accuracy_layers.append(layer)
else:
hidden_layers.layer.add().CopyFrom(layer)
if len(layer.bottom) == 1 and len(
layer.top) == 1 and layer.bottom[0] == layer.top[0]:
pass
else:
for top in layer.top:
tops.append(top)
for bottom in layer.bottom:
bottoms[bottom] = True
if train_data_layer is None:
assert val_data_layer is None, 'cannot specify a test data layer without a train data layer'
assert len(loss_layers) > 0, 'must specify a loss layer'
network_outputs = []
for name in tops:
if name not in bottoms:
network_outputs.append(name)
assert len(network_outputs), 'network must have an output'
# Update num_output for any output InnerProduct layers automatically
for layer in hidden_layers.layer:
if layer.type == 'InnerProduct':
for top in layer.top:
if top in network_outputs:
layer.inner_product_param.num_output = len(
self.get_labels())
break
### Write train_val file
train_val_network = caffe_pb2.NetParameter()
# data layers
if train_data_layer is not None:
if train_data_layer.HasField('data_param'):
assert not train_data_layer.data_param.HasField(
'source'), "don't set the data_param.source"
assert not train_data_layer.data_param.HasField(
'backend'), "don't set the data_param.backend"
max_crop_size = min(self.dataset.image_dims[0],
self.dataset.image_dims[1])
if self.crop_size:
assert self.crop_size <= max_crop_size, 'crop_size is larger than the image size'
train_data_layer.transform_param.crop_size = self.crop_size
elif train_data_layer.transform_param.HasField('crop_size'):
cs = train_data_layer.transform_param.crop_size
if cs > max_crop_size:
# don't throw an error here
cs = max_crop_size
train_data_layer.transform_param.crop_size = cs
self.crop_size = cs
train_val_network.layer.add().CopyFrom(train_data_layer)
train_data_layer = train_val_network.layer[-1]
if val_data_layer is not None and has_val_set:
if val_data_layer.HasField('data_param'):
assert not val_data_layer.data_param.HasField(
'source'), "don't set the data_param.source"
assert not val_data_layer.data_param.HasField(
'backend'), "don't set the data_param.backend"
if self.crop_size:
# use our error checking from the train layer
val_data_layer.transform_param.crop_size = self.crop_size
train_val_network.layer.add().CopyFrom(val_data_layer)
val_data_layer = train_val_network.layer[-1]
else:
train_data_layer = train_val_network.layer.add(type='Data',
name='data')
train_data_layer.top.append('data')
train_data_layer.top.append('label')
train_data_layer.include.add(phase=caffe_pb2.TRAIN)
train_data_layer.data_param.batch_size = constants.DEFAULT_BATCH_SIZE
if self.crop_size:
train_data_layer.transform_param.crop_size = self.crop_size
if has_val_set:
val_data_layer = train_val_network.layer.add(type='Data',
name='data')
val_data_layer.top.append('data')
val_data_layer.top.append('label')
val_data_layer.include.add(phase=caffe_pb2.TEST)
val_data_layer.data_param.batch_size = constants.DEFAULT_BATCH_SIZE
if self.crop_size:
val_data_layer.transform_param.crop_size = self.crop_size
train_data_layer.data_param.source = self.dataset.path(
self.dataset.train_db_task().db_name)
train_data_layer.data_param.backend = caffe_pb2.DataParameter.LMDB
if val_data_layer is not None and has_val_set:
val_data_layer.data_param.source = self.dataset.path(
self.dataset.val_db_task().db_name)
val_data_layer.data_param.backend = caffe_pb2.DataParameter.LMDB
if self.use_mean:
mean_pixel = None
with open(self.dataset.path(
self.dataset.train_db_task().mean_file)) as f:
blob = caffe_pb2.BlobProto()
blob.MergeFromString(f.read())
mean = np.reshape(blob.data, (
self.dataset.image_dims[2],
self.dataset.image_dims[0],
self.dataset.image_dims[1],
))
mean_pixel = mean.mean(1).mean(1)
for value in mean_pixel:
train_data_layer.transform_param.mean_value.append(value)
if val_data_layer is not None and has_val_set:
for value in mean_pixel:
val_data_layer.transform_param.mean_value.append(value)
if self.batch_size:
train_data_layer.data_param.batch_size = self.batch_size
if val_data_layer is not None and has_val_set:
val_data_layer.data_param.batch_size = self.batch_size
else:
if not train_data_layer.data_param.HasField('batch_size'):
train_data_layer.data_param.batch_size = constants.DEFAULT_BATCH_SIZE
if val_data_layer is not None and has_val_set and not val_data_layer.data_param.HasField(
'batch_size'):
val_data_layer.data_param.batch_size = constants.DEFAULT_BATCH_SIZE
# hidden layers
train_val_network.MergeFrom(hidden_layers)
# output layers
train_val_network.layer.extend(loss_layers)
train_val_network.layer.extend(accuracy_layers)
with open(self.path(self.train_val_file), 'w') as outfile:
text_format.PrintMessage(train_val_network, outfile)
### Write deploy file
deploy_network = caffe_pb2.NetParameter()
# input
deploy_network.input.append('data')
deploy_network.input_dim.append(1)
deploy_network.input_dim.append(self.dataset.image_dims[2])
if self.crop_size:
deploy_network.input_dim.append(self.crop_size)
deploy_network.input_dim.append(self.crop_size)
else:
deploy_network.input_dim.append(self.dataset.image_dims[0])
deploy_network.input_dim.append(self.dataset.image_dims[1])
# hidden layers
deploy_network.MergeFrom(hidden_layers)
# output layers
if loss_layers[-1].type == 'SoftmaxWithLoss':
prob_layer = deploy_network.layer.add(type='Softmax', name='prob')
prob_layer.bottom.append(network_outputs[-1])
prob_layer.top.append('prob')
with open(self.path(self.deploy_file), 'w') as outfile:
text_format.PrintMessage(deploy_network, outfile)
### Write solver file
solver = caffe_pb2.SolverParameter()
# get enum value for solver type
solver.solver_type = getattr(solver, self.solver_type)
solver.net = self.train_val_file
# Set CPU/GPU mode
if config_value('caffe_root')['cuda_enabled'] and \
bool(config_value('gpu_list')):
solver.solver_mode = caffe_pb2.SolverParameter.GPU
else:
solver.solver_mode = caffe_pb2.SolverParameter.CPU
solver.snapshot_prefix = self.snapshot_prefix
# Epochs -> Iterations
train_iter = int(
math.ceil(
float(self.dataset.train_db_task().entries_count) /
train_data_layer.data_param.batch_size))
solver.max_iter = train_iter * self.train_epochs
snapshot_interval = self.snapshot_interval * train_iter
if 0 < snapshot_interval <= 1:
solver.snapshot = 1 # don't round down
elif 1 < snapshot_interval < solver.max_iter:
solver.snapshot = int(snapshot_interval)
else:
solver.snapshot = 0 # only take one snapshot at the end
if has_val_set and self.val_interval:
solver.test_iter.append(
int(
math.ceil(
float(self.dataset.val_db_task().entries_count) /
val_data_layer.data_param.batch_size)))
val_interval = self.val_interval * train_iter
if 0 < val_interval <= 1:
solver.test_interval = 1 # don't round down
elif 1 < val_interval < solver.max_iter:
solver.test_interval = int(val_interval)
else:
solver.test_interval = solver.max_iter # only test once at the end
# Learning rate
solver.base_lr = self.learning_rate
solver.lr_policy = self.lr_policy['policy']
scale = float(solver.max_iter) / 100.0
if solver.lr_policy == 'fixed':
pass
elif solver.lr_policy == 'step':
# stepsize = stepsize * scale
solver.stepsize = int(
math.ceil(float(self.lr_policy['stepsize']) * scale))
solver.gamma = self.lr_policy['gamma']
elif solver.lr_policy == 'multistep':
for value in self.lr_policy['stepvalue']:
# stepvalue = stepvalue * scale
solver.stepvalue.append(int(math.ceil(float(value) * scale)))
solver.gamma = self.lr_policy['gamma']
elif solver.lr_policy == 'exp':
# gamma = gamma^(1/scale)
solver.gamma = math.pow(self.lr_policy['gamma'], 1.0 / scale)
elif solver.lr_policy == 'inv':
# gamma = gamma / scale
solver.gamma = self.lr_policy['gamma'] / scale
solver.power = self.lr_policy['power']
elif solver.lr_policy == 'poly':
solver.power = self.lr_policy['power']
elif solver.lr_policy == 'sigmoid':
# gamma = -gamma / scale
solver.gamma = -1.0 * self.lr_policy['gamma'] / scale
# stepsize = stepsize * scale
solver.stepsize = int(
math.ceil(float(self.lr_policy['stepsize']) * scale))
else:
raise Exception('Unknown lr_policy: "%s"' % solver.lr_policy)
# go with the suggested defaults
if solver.solver_type != solver.ADAGRAD:
solver.momentum = 0.9
solver.weight_decay = 0.0005
# Display 8x per epoch, or once per 5000 images, whichever is more frequent
solver.display = max(
1,
min(
int(
math.floor(
float(solver.max_iter) / (self.train_epochs * 8))),
int(math.ceil(5000.0 /
train_data_layer.data_param.batch_size))))
if self.random_seed is not None:
solver.random_seed = self.random_seed
with open(self.path(self.solver_file), 'w') as outfile:
text_format.PrintMessage(solver, outfile)
self.solver = solver # save for later
return True
def task_arguments(self, resources, env):
if config_value('torch_root') == '<PATHS>':
torch_bin = 'th'
else:
torch_bin = os.path.join(config_value('torch_root'), 'bin', 'th')
dataset_backend = self.dataset.train_db_task().backend
assert dataset_backend=='lmdb' or dataset_backend=='hdf5'
args = [torch_bin,
os.path.join(os.path.dirname(os.path.dirname(digits.__file__)),'tools','torch','main.lua'),
'--network=%s' % self.model_file.split(".")[0],
'--epoch=%d' % int(self.train_epochs),
'--networkDirectory=%s' % self.job_dir,
'--save=%s' % self.job_dir,
'--snapshotPrefix=%s' % self.snapshot_prefix,
'--snapshotInterval=%s' % self.snapshot_interval,
'--learningRate=%s' % self.learning_rate,
'--policy=%s' % str(self.lr_policy['policy']),
'--dbbackend=%s' % dataset_backend
]
if self.batch_size is not None:
args.append('--batchSize=%d' % self.batch_size)
if isinstance(self.dataset, ImageClassificationDatasetJob):
args.append('--train=%s' % self.dataset.path(constants.TRAIN_DB))
if os.path.exists(self.dataset.path(constants.VAL_DB)) and self.val_interval > 0:
args.append('--validation=%s' % self.dataset.path(constants.VAL_DB))
args.append('--mean=%s' % self.dataset.path(constants.MEAN_FILE_IMAGE))
args.append('--labels=%s' % self.dataset.path(self.dataset.labels_file))
elif isinstance(self.dataset, dataset.GenericImageDatasetJob):
train_image_db = None
train_labels_db = None
val_image_db = None
val_labels_db = None
for task in self.dataset.tasks:
if task.purpose == 'Training Images':
train_image_db = task
if task.purpose == 'Training Labels':
train_labels_db = task
if task.purpose == 'Validation Images':
val_image_db = task
if task.purpose == 'Validation Labels':
val_labels_db = task
assert train_image_db is not None, 'Training images are required'
args.append('--train=%s' % train_image_db.path(train_image_db.database))
if train_labels_db:
args.append('--train_labels=%s' % train_labels_db.path(train_labels_db.database))
if val_image_db:
args.append('--validation=%s' % val_image_db.path(val_image_db.database))
if val_labels_db:
args.append('--validation_labels=%s' % val_labels_db.path(val_labels_db.database))
if self.use_mean != 'none':
assert self.dataset.mean_file.endswith('.binaryproto'), 'Mean subtraction required but dataset has no mean file in .binaryproto format'
blob = caffe_pb2.BlobProto()
with open(task.path(self.dataset.mean_file),'rb') as infile:
blob.ParseFromString(infile.read())
data = np.array(blob.data, dtype=np.uint8).reshape(blob.channels, blob.height, blob.width)
if blob.channels == 3:
# converting from BGR to RGB
data = data[[2,1,0],...] # channel swap
# convert to (height, width, channels)
data = data.transpose((1,2,0))
else:
assert blob.channels == 1
# convert to (height, width)
data = data[0]
# save to file
filename = os.path.join(self.job_dir, constants.MEAN_FILE_IMAGE)
image = PIL.Image.fromarray(data)
image.save(filename)
args.append('--mean=%s' % filename)
#learning rate policy input parameters
if self.lr_policy['policy'] == 'fixed':
pass
elif self.lr_policy['policy'] == 'step':
args.append('--gamma=%s' % self.lr_policy['gamma'])
args.append('--stepvalues=%s' % self.lr_policy['stepsize'])
elif self.lr_policy['policy'] == 'multistep':
args.append('--stepvalues=%s' % self.lr_policy['stepvalue'])
args.append('--gamma=%s' % self.lr_policy['gamma'])
elif self.lr_policy['policy'] == 'exp':
args.append('--gamma=%s' % self.lr_policy['gamma'])
elif self.lr_policy['policy'] == 'inv':
args.append('--gamma=%s' % self.lr_policy['gamma'])
args.append('--power=%s' % self.lr_policy['power'])
elif self.lr_policy['policy'] == 'poly':
args.append('--power=%s' % self.lr_policy['power'])
elif self.lr_policy['policy'] == 'sigmoid':
args.append('--stepvalues=%s' % self.lr_policy['stepsize'])
args.append('--gamma=%s' % self.lr_policy['gamma'])
if self.shuffle:
args.append('--shuffle=yes')
if self.crop_size:
args.append('--crop=yes')
args.append('--croplen=%d' % self.crop_size)
if self.use_mean == 'pixel':
args.append('--subtractMean=pixel')
elif self.use_mean == 'image':
args.append('--subtractMean=image')
else:
args.append('--subtractMean=none')
if self.random_seed is not None:
args.append('--seed=%s' % self.random_seed)
if self.solver_type == 'NESTEROV':
args.append('--optimization=nag')
if self.solver_type == 'ADAGRAD':
args.append('--optimization=adagrad')
if self.solver_type == 'SGD':
args.append('--optimization=sgd')
if self.val_interval > 0:
args.append('--interval=%s' % self.val_interval)
if 'gpus' in resources:
identifiers = []
for identifier, value in resources['gpus']:
identifiers.append(int(identifier))
if len(identifiers) == 1:
# only one device must be visible to the th process
# to prevent Torch from loading libraries on all GPUs
env['CUDA_VISIBLE_DEVICES'] = str(identifiers[0])
args.append('--devid=1')
elif len(identifiers) > 1:
raise NotImplementedError("Multi-GPU with Torch not supported yet")
else:
# switch to CPU mode
args.append('--type=float')
self.trained_on_cpu = True
if self.pretrained_model:
args.append('--weights=%s' % self.path(self.pretrained_model))
return args
builder.add_inner_product(name="ip2",
W=W,
b=b,
input_channels=64,
output_channels=10,
has_bias=True,
input_name="ip1_output",
output_name="ip2_output")
builder.add_softmax(name="softmax",
input_name="ip2_output",
output_name="labelProbs")
import caffe_pb2
mean_image = caffe_pb2.BlobProto()
mean_image.ParseFromString(open("mean.binaryproto", "rb").read())
mean_image = np.array(mean_image.data)
builder.spec.neuralNetworkClassifier.preprocessing[
0].meanImage.meanImage.extend(mean_image)
mlmodel = coremltools.models.MLModel(builder.spec)
mlmodel.short_description = "cifar10_quick"
mlmodel.author = "https://github.com/BVLC/caffe/tree/master/examples/cifar10"
mlmodel.license = "https://github.com/BVLC/caffe/blob/master/LICENSE"
mlmodel.input_description["image"] = "The input image"
mlmodel.output_description["labelProbs"] = "The predicted probabilities"
mlmodel.output_description["label"] = "The class with the highest score"
print(f'export {name}')
layer = net.layer.add()
layer.name = name
layer.type = 'Convolution'
layer.bottom.append(bottom)
layer.top.append(name)
layer.convolution_param.num_output = module.out_channels
layer.convolution_param.bias_term = module.bias is not None
layer.convolution_param.pad.append(module.kernel_size[0] == 3)
layer.convolution_param.kernel_size.append(module.kernel_size[0])
layer.convolution_param.stride.append(module.stride[0])
if residual and 'conv1' in name:
skip_bottom = bottom # first bottom of Eltwise layer
bottom = name
layer.blobs.append(pb.BlobProto())
layer.blobs[-1].shape.dim.append(module.weight.size(0))
layer.blobs[-1].shape.dim.append(module.weight.size(1))
layer.blobs[-1].shape.dim.append(module.weight.size(2))
layer.blobs[-1].shape.dim.append(module.weight.size(3))
for data in module.weight.detach().cpu().numpy().flatten():
layer.blobs[-1].data.append(data)
if module.bias is not None:
layer.blobs.append(pb.BlobProto())
layer.blobs[-1].shape.dim.append(module.bias.size(0))
for data in module.bias.detach().cpu().numpy().flatten():
layer.blobs[-1].data.append(data)
elif isinstance(module, torch.nn.BatchNorm2d): # BatchNorm and Scale
print(f'export {name}')
bn_layer = net.layer.add()
bn_layer.name = name
def load_weights(self, weightfile):
if self.blocks[0].has_key('mean_file'):
import caffe_pb2
mean_file = self.blocks[0]['mean_file']
mean_file = mean_file.strip('"')
self.has_mean = True
blob = caffe_pb2.BlobProto()
blob.ParseFromString(open(mean_file, 'rb').read())
mean_img = torch.from_numpy(np.array(blob.data)).float()
channels = int(self.blocks[0]['channels'])
height = int(self.blocks[0]['height'])
width = int(self.blocks[0]['width'])
mean_img = mean_img.view(channels, height, width)
self.register_buffer('mean_img',
torch.zeros(channels, height, width))
self.mean_img.copy_(mean_img)
fp = open(weightfile, 'rb')
header = np.fromfile(fp, count=4, dtype=np.int32)
self.header = torch.from_numpy(header)
self.seen = self.header[3]
buf = np.fromfile(fp, dtype=np.float32)
fp.close()
start = 0
ind = -2
is_first = True
for block in self.blocks:
if start >= buf.size:
break
ind = ind + 1
if block['type'] == 'net':
continue
elif block['type'] == 'convolutional':
model = self.models[ind]
batch_normalize = int(block['batch_normalize'])
if batch_normalize:
start = load_conv_bn(buf, start, model[0], model[1])
else:
start = load_conv(buf, start, model[0])
elif block['type'] == 'connected':
model = self.models[ind]
if block['activation'] != 'linear':
if is_first:
start = load_fc(buf, start, model[1])
is_first = False
else:
start = load_fc(buf, start, model[0])
else:
if is_first:
start = load_fc(buf, start, model[1])
is_first = False
else:
start = load_fc(buf, start, model)
elif block['type'] == 'maxpool':
pass
elif block['type'] == 'reorg':
pass
elif block['type'] == 'route':
pass
elif block['type'] == 'shortcut':
pass
elif block['type'] == 'avgpool':
pass
elif block['type'] == 'softmax':
pass
elif block['type'] == 'cost':
pass
elif block['type'] == 'region':
pass
elif block['type'] == 'dropout':
pass
else:
print('unknown type %s' % (block['type']))
def load_weights(self, caffemodel):
if self.net_info['props'].has_key('mean_file'):
import caffe_pb2
self.has_mean = True
mean_file = self.net_info['props']['mean_file']
blob = caffe_pb2.BlobProto()
blob.ParseFromString(open(mean_file, 'rb').read())
mean_img = torch.from_numpy(np.array(blob.data)).float()
if self.net_info['props'].has_key('input_shape'):
channels = int(self.net_info['props']['input_shape']['dim'][1])
height = int(self.net_info['props']['input_shape']['dim'][2])
width = int(self.net_info['props']['input_shape']['dim'][3])
else:
channels = int(self.net_info['props']['input_dim'][1])
height = int(self.net_info['props']['input_dim'][2])
width = int(self.net_info['props']['input_dim'][3])
mean_img = mean_img.view(channels, height, width) #.mean(0)
#mean_img = mean_img.repeat(3, 1, 1)
self.register_buffer('mean_img',
torch.zeros(channels, height, width))
self.mean_img.copy_(mean_img)
model = parse_caffemodel(caffemodel)
layers = model.layer
if len(layers) == 0:
print('Using V1LayerParameter')
layers = model.layers
lmap = {}
for l in layers:
lmap[l.name] = l
layers = self.net_info['layers']
layer_num = len(layers)
i = 0
while i < layer_num:
layer = layers[i]
lname = layer['name']
ltype = layer['type']
if ltype == 'Convolution':
convolution_param = layer['convolution_param']
bias = True
if convolution_param.has_key(
'bias_term'
) and convolution_param['bias_term'] == 'false':
bias = False
self.models[lname].weight.data.copy_(
torch.from_numpy(np.array(lmap[lname].blobs[0].data)))
if bias and len(lmap[lname].blobs) > 1:
print('convlution %s has bias' % lname)
self.models[lname].bias.data.copy_(
torch.from_numpy(np.array(lmap[lname].blobs[1].data)))
i = i + 1
elif ltype == 'BatchNorm':
scale_layer = layers[i + 1]
self.models[lname].running_mean.copy_(
torch.from_numpy(
np.array(lmap[lname].blobs[0].data) /
lmap[lname].blobs[2].data[0]))
self.models[lname].running_var.copy_(
torch.from_numpy(
np.array(lmap[lname].blobs[1].data) /
lmap[lname].blobs[2].data[0]))
self.models[lname].weight.data.copy_(
torch.from_numpy(
np.array(lmap[scale_layer['name']].blobs[0].data)))
self.models[lname].bias.data.copy_(
torch.from_numpy(
np.array(lmap[scale_layer['name']].blobs[1].data)))
i = i + 2
elif ltype == 'InnerProduct':
if type(self.models[lname]) == nn.Sequential:
self.models[lname][1].weight.data.copy_(
torch.from_numpy(np.array(lmap[lname].blobs[0].data)))
if len(lmap[lname].blobs) > 1:
self.models[lname][1].bias.data.copy_(
torch.from_numpy(
np.array(lmap[lname].blobs[1].data)))
else:
self.models[lname].weight.data.copy_(
torch.from_numpy(np.array(lmap[lname].blobs[0].data)))
if len(lmap[lname].blobs) > 1:
self.models[lname].bias.data.copy_(
torch.from_numpy(
np.array(lmap[lname].blobs[1].data)))
i = i + 1
elif ltype == 'Pooling' or ltype == 'Eltwise' or ltype == 'ReLU' or ltype == 'Region':
i = i + 1
else:
print('load_weights: unknown type %s' % ltype)
i = i + 1
