def getPeakFeatures():
net = DecafNet()
features = numpy.zeros((number_sequences, feature_length))
labels = numpy.zeros((number_sequences, 1))
counter = 0
# Maybe sort them
for participant in os.listdir(os.path.join(data_dir, image_dir)):
for sequence in os.listdir(
os.path.join(data_dir, image_dir, participant)):
if sequence != ".DS_Store":
image_files = sorted(
os.listdir(
os.path.join(data_dir, image_dir, participant,
sequence)))
image_file = image_files[-1]
print counter, image_file
imarray = cv2.imread(
os.path.join(data_dir, image_dir, participant, sequence,
image_file))
imarray = cv2.cvtColor(imarray, cv2.COLOR_BGR2GRAY)
scores = net.classify(imarray, center_only=True)
features[counter] = net.feature(feature_level) #.flatten()
label_file = open(
os.path.join(data_dir, label_dir, participant, sequence,
image_file[:-4] + "_emotion.txt"))
labels[counter] = eval(label_file.read())
label_file.close()
counter += 1
numpy.save("featuresPeak5", features)
numpy.save("labelsPeak5", labels)
def full_check_decaf(win_slide=5, win_size=1024, blob_name='fc6_cudanet_out'):
from decaf.scripts.imagenet import DecafNet
net = DecafNet()
clf = joblib.load("420_decaf/classifier_decaf.pkl")
g_raster = gdal.Open('20-21-22-part2.tif') # test.tif
# plt.axis('off')
# f, axarr = plt.subplots(n, n)
result = {}
cols = range(0, g_raster.RasterXSize - win_size, win_slide)
rows = range(0, g_raster.RasterYSize - win_size, win_slide)
full = len(rows) * len(cols)
count = 0
pbar = progressbar.ProgressBar(maxval=full).start()
for i in range(0, g_raster.RasterXSize - win_size, win_slide):
for j in range(0, g_raster.RasterYSize - win_size, win_slide):
img = get_sample(g_raster, i, j, win_size)
net.classify(img, True)
tmp = net.feature(blob_name) #与训练时候保持一致
result[(j,i)] = clf.predict(tmp)
if result[(j,i)] == 2:
io.imsave("420_decaf/slide_target/%s_%s_%s_%s.png" % (j, i, j+win_size, i+win_size), img)
pbar.update(count+1)
count = count + 1
pbar.finish()
arr = np.ones((len(rows), len(cols)))
for k, v in result.items():
if v != 0 and v[0] == 2:
arr[k[0]/win_slide, k[1]/win_slide] = v[0]
return arr
def getMoreFeatures():
net = DecafNet()
features = []
labels = []
counter = 0
for participant in os.listdir(os.path.join(data_dir, image_dir)):
for sequence in os.listdir(
os.path.join(data_dir, image_dir, participant)):
if sequence != ".DS_Store":
image_files = sorted(
os.listdir(
os.path.join(data_dir, image_dir, participant,
sequence)))
cutoff = len(image_files) / 2
image_files = image_files[cutoff::]
label_file = open(
os.path.join(data_dir, label_dir, participant, sequence,
image_files[-1][:-4] + "_emotion.txt"))
label = eval(label_file.read())
label_file.close()
for image_file in image_files:
print counter, image_file
imarray = numpy.asarray(
Image.open(
os.path.join(data_dir, image_dir, participant,
sequence, image_file)))
scores = net.classify(imarray, center_only=True)
features.append(net.feature(feature_level))
labels.append(label)
counter += 1
numpy.save("featuresMore", numpy.array(features))
numpy.save("labelsMore", numpy.array(labels))
def getMoreFeatures():
net = DecafNet()
features = []
labels = []
counter = 0
for participant in os.listdir(os.path.join(data_dir,image_dir)):
for sequence in os.listdir(os.path.join(data_dir,image_dir, participant)):
if sequence != ".DS_Store":
image_files = sorted(os.listdir(os.path.join(data_dir,image_dir, participant,sequence)))
cutoff = len(image_files)/2
image_files = image_files[cutoff::]
label_file = open(os.path.join(data_dir,label_dir, participant,sequence,image_files[-1][:-4]+"_emotion.txt"))
label = eval(label_file.read())
label_file.close()
for image_file in image_files:
print counter, image_file
imarray = numpy.asarray(Image.open(os.path.join(data_dir,image_dir, participant,sequence,image_file)))
scores = net.classify(imarray, center_only=True)
features.append(net.feature(feature_level))
labels.append(label)
counter += 1
numpy.save("featuresMore",numpy.array(features))
numpy.save("labelsMore",numpy.array(labels))
def getPeakFaceFeatures():
net = DecafNet()
cascade = cv2.CascadeClassifier('haarcascade_frontalface_alt2.xml')
features = numpy.zeros((number_sequences,feature_length))
labels = numpy.zeros((number_sequences,1))
counter = 0
# Maybe sort them
for participant in os.listdir(os.path.join(data_dir,image_dir)):
for sequence in os.listdir(os.path.join(data_dir,image_dir, participant)):
if sequence != ".DS_Store":
image_files = sorted(os.listdir(os.path.join(data_dir,image_dir, participant,sequence)))
image_file = image_files[-1]
print counter, image_file
imarray = cv2.imread(os.path.join(data_dir,image_dir, participant,sequence,image_file))
imarray = cv2.cvtColor(imarray,cv2.COLOR_BGR2GRAY)
rects = cascade.detectMultiScale(imarray, 1.3, 3, cv2.cv.CV_HAAR_SCALE_IMAGE, (150,150))
if len(rects) > 0:
facerect=rects[0]
imarray = imarray[facerect[1]:facerect[1]+facerect[3], facerect[0]:facerect[0]+facerect[2]]
scores = net.classify(imarray, center_only=True)
features[counter] = net.feature(feature_level).flatten()
label_file = open(os.path.join(data_dir,label_dir, participant,sequence,image_file[:-4]+"_emotion.txt"))
labels[counter] = eval(label_file.read())
label_file.close()
counter += 1
numpy.save("featuresPeakFace5",features)
numpy.save("labelsPeakFace5",labels)
def label(data_path, number_of_labels=1, equal_weights=True):
"""
Create annotation files for directories of pictures.
Number of labels returned can be adjusted.
Equal weighting of predictions.
"""
net = DecafNet()
entities = os.listdir(data_path)
for entity in entities:
labels = []
pictures = os.listdir(os.path.join(data_path,entity))
for picture in pictures:
imarray = numpy.array(Image.open(os.path.join(data_path,entity,picture)))
scores = net.classify(imarray)
predictions = net.top_k_prediction(scores,number_of_labels) # Format:([confidence],[labels])
labels.extend(predictions[1])
if equal_weights:
prediction = max(set(labels),key=labels.count)
elif confidence_weights:
pass
with open("label.txt","w") as opened_file:
opened_file.write(prediction)
def __init__(
self,
layer_name,
model_path='dist/decaf-release/model/imagenet.decafnet.epoch90',
meta_path='dist/decaf-release/model/imagenet.decafnet.meta'):
self.layer_name = layer_name
self.net = DecafNet(model_path, meta_path)
self.transforms = [NopTransform()]
def fit(self, X=None, y=None):
from decaf.scripts.imagenet import DecafNet # soft dep
if self.net_ is None:
self.net_ = DecafNet(
self.pretrained_params,
self.pretrained_meta,
)
return self
def __init__(self, decaf_folder=None, classifer_file=None):
if decaf_folder is None:
decaf_folder = '../models/imagenet_pretrained/'
if classifer_file is None:
classifer_file = "../models/lg_classifier_public"
self.net = DecafNet(
path.join(decaf_folder, 'imagenet.decafnet.epoch90'),
path.join(decaf_folder, 'imagenet.decafnet.meta'))
self.feat_layer = 'fc6_cudanet_out'
self.classifier = cPickle.load(open(classifer_file, "r"))
class AVIClassifier(object):
def __init__(self, decaf_folder = None, classifer_file = None):
if decaf_folder is None:
decaf_folder = '../models/imagenet_pretrained/'
if classifer_file is None:
classifer_file = "../models/lg_classifier_public"
self.net = DecafNet(path.join(decaf_folder, 'imagenet.decafnet.epoch90'),
path.join(decaf_folder, 'imagenet.decafnet.meta'))
self.feat_layer = 'fc6_cudanet_out'
self.classifier = cPickle.load(open(classifer_file, "r"))
def predict(self, img):
im = img_as_ubyte(color.rgb2gray(img))
scores = self.net.classify(im, center_only = True)
feats = self.net.feature(self.feat_layer).flatten()
defect_probs = self.classifier.predict_proba(feats)
return sorted(zip(self.classifier.classes_, defect_probs[0]), key = lambda (cls, prob): prob, reverse=True)
def class_names(self):
return self.classifier.classes_
def __init__(self, decaf_folder = None, classifer_file = None):
if decaf_folder is None:
decaf_folder = '../models/imagenet_pretrained/'
if classifer_file is None:
classifer_file = "../models/lg_classifier_public"
self.net = DecafNet(path.join(decaf_folder, 'imagenet.decafnet.epoch90'),
path.join(decaf_folder, 'imagenet.decafnet.meta'))
self.feat_layer = 'fc6_cudanet_out'
self.classifier = cPickle.load(open(classifer_file, "r"))
def __init__(self, model_spec_filename, model_filename=None,\
wnid_words_filename=None, center_only=False, wnid_subset = []):
"""
*** PRIVATE CONSTRUCTOR ***
"""
# the following is just an hack to allow retro-compatibility
# with existing code
if isinstance(model_spec_filename, NetworkDecafParams):
params = model_spec_filename
model_spec_filename = params.model_spec_filename
model_filename = params.model_filename
wnid_words_filename = params.wnid_words_filename
center_only = params.center_only
wnid_subset = params.wnid_subset
if wnid_subset != []:
print 'Warning: subset of labels not supported yet'
else:
assert isinstance(model_spec_filename, str)
assert model_filename != None
assert wnid_words_filename != None
# load Decaf model
self.net_ = DecafNet(model_filename, model_spec_filename)
self.center_only_ = center_only
# build a dictionary label --> description
self.dict_label_desc_ = {}
dict_desc_label = {}
fd = open(wnid_words_filename)
for line in fd:
temp = line.strip().split('\t')
wnid = temp[1].strip()
self.dict_label_desc_[wnid] = temp[2].strip()
dict_desc_label[temp[2].split(',')[0]] = wnid
fd.close()
# build a dictionary label --> label_id
self.dict_label_id_ = {}
self.labels_ = []
for i, desc in enumerate(self.net_.label_names):
self.dict_label_id_[dict_desc_label[desc]] = i
self.labels_.append(dict_desc_label[desc])
# Load the mean vector from file
# mean of 3 channels
self.net_.mean_img = np.mean(np.mean(self.net_._data_mean, axis=1),
axis=0)
def getPeakFaceFeatures():
net = DecafNet()
cascade = cv2.CascadeClassifier('haarcascade_frontalface_alt2.xml')
features = numpy.zeros((number_sequences, feature_length))
labels = numpy.zeros((number_sequences, 1))
counter = 0
# Maybe sort them
for participant in os.listdir(os.path.join(data_dir, image_dir)):
for sequence in os.listdir(
os.path.join(data_dir, image_dir, participant)):
if sequence != ".DS_Store":
image_files = sorted(
os.listdir(
os.path.join(data_dir, image_dir, participant,
sequence)))
image_file = image_files[-1]
print counter, image_file
imarray = cv2.imread(
os.path.join(data_dir, image_dir, participant, sequence,
image_file))
imarray = cv2.cvtColor(imarray, cv2.COLOR_BGR2GRAY)
rects = cascade.detectMultiScale(imarray, 1.3, 3,
cv2.cv.CV_HAAR_SCALE_IMAGE,
(150, 150))
if len(rects) > 0:
facerect = rects[0]
imarray = imarray[facerect[1]:facerect[1] + facerect[3],
facerect[0]:facerect[0] + facerect[2]]
scores = net.classify(imarray, center_only=True)
features[counter] = net.feature(feature_level).flatten()
label_file = open(
os.path.join(data_dir, label_dir, participant, sequence,
image_file[:-4] + "_emotion.txt"))
labels[counter] = eval(label_file.read())
label_file.close()
counter += 1
numpy.save("featuresPeakFace5", features)
numpy.save("labelsPeakFace5", labels)
class AVIClassifier(object):
def __init__(self, decaf_folder=None, classifer_file=None):
if decaf_folder is None:
decaf_folder = '../models/imagenet_pretrained/'
if classifer_file is None:
classifer_file = "../models/lg_classifier_public"
self.net = DecafNet(
path.join(decaf_folder, 'imagenet.decafnet.epoch90'),
path.join(decaf_folder, 'imagenet.decafnet.meta'))
self.feat_layer = 'fc6_cudanet_out'
self.classifier = cPickle.load(open(classifer_file, "r"))
def predict(self, img):
im = img_as_ubyte(color.rgb2gray(img))
scores = self.net.classify(im, center_only=True)
feats = self.net.feature(self.feat_layer).flatten()
defect_probs = self.classifier.predict_proba(feats)
return sorted(zip(self.classifier.classes_, defect_probs[0]),
key=lambda (cls, prob): prob,
reverse=True)
def class_names(self):
return self.classifier.classes_
def __init__(
self,
feature_layer='fc7_cudanet_out',
pretrained_params='imagenet.decafnet.epoch90',
pretrained_meta='imagenet.decafnet.meta',
center_only=True
):
"""
:param feature_layer: The ConvNet layer that's used for
feature extraction. Defaults to
`fc7_cudanet_out`. A description of all
available layers for the
ImageNet-1k-pretrained ConvNet is found
in the DeCAF wiki. They are:
- `pool5_cudanet_out`
- `fc6_cudanet_out`
- `fc6_neuron_cudanet_out`
- `fc7_cudanet_out`
- `fc7_neuron_cudanet_out`
:param pretrained_params: This must point to the file with the
pretrained parameters. Defaults to
`imagenet.decafnet.epoch90`. For
the ImageNet-1k-pretrained ConvNet
this file can be obtained from here:
http://www.eecs.berkeley.edu/~jiayq/decaf_pretrained/
:param pretrained_meta: Similar to `pretrained_params`, this
must file to the file with the
pretrained parameters' metadata.
Defaults to `imagenet.decafnet.meta`.
:param center_only: Use the center patch of the image only
when extracting features. If `False`, use
four corners, the image center and flipped
variants and average a total of 10 feature
vectors, which will usually yield better
results. Defaults to `True`.
"""
super(ConvNetFeatureExtractor, self).__init__()
self.feature_layer = feature_layer
self.pretrained_params = pretrained_params
self.pretrained_meta = pretrained_meta
self.center_only = center_only
self.convnet = DecafNet(
self.pretrained_params,
self.pretrained_meta
)
def getPeakFeatures():
net = DecafNet()
features = numpy.zeros((number_sequences,feature_length))
labels = numpy.zeros((number_sequences,1))
counter = 0
# Maybe sort them
for participant in os.listdir(os.path.join(data_dir,image_dir)):
for sequence in os.listdir(os.path.join(data_dir,image_dir, participant)):
if sequence != ".DS_Store":
image_files = sorted(os.listdir(os.path.join(data_dir,image_dir, participant,sequence)))
image_file = image_files[-1]
print counter, image_file
imarray = cv2.imread(os.path.join(data_dir,image_dir, participant,sequence,image_file))
imarray = cv2.cvtColor(imarray,cv2.COLOR_BGR2GRAY)
scores = net.classify(imarray, center_only=True)
features[counter] = net.feature(feature_level)#.flatten()
label_file = open(os.path.join(data_dir,label_dir, participant,sequence,image_file[:-4]+"_emotion.txt"))
labels[counter] = eval(label_file.read())
label_file.close()
counter += 1
numpy.save("featuresPeak5",features)
numpy.save("labelsPeak5",labels)
def __init__(self, model_spec_filename, model_filename=None,\
wnid_words_filename=None, center_only=False, wnid_subset = []):
"""
*** PRIVATE CONSTRUCTOR ***
"""
# the following is just an hack to allow retro-compatibility
# with existing code
if isinstance(model_spec_filename, NetworkDecafParams):
params = model_spec_filename
model_spec_filename = params.model_spec_filename
model_filename = params.model_filename
wnid_words_filename = params.wnid_words_filename
center_only = params.center_only
wnid_subset = params.wnid_subset
if wnid_subset!=[]:
print 'Warning: subset of labels not supported yet'
else:
assert isinstance(model_spec_filename, str)
assert model_filename != None
assert wnid_words_filename != None
# load Decaf model
self.net_ = DecafNet(model_filename, model_spec_filename)
self.center_only_ = center_only
# build a dictionary label --> description
self.dict_label_desc_ = {}
dict_desc_label = {}
fd = open(wnid_words_filename)
for line in fd:
temp = line.strip().split('\t')
wnid = temp[1].strip()
self.dict_label_desc_[wnid] = temp[2].strip()
dict_desc_label[temp[2].split(',')[0]] = wnid
fd.close()
# build a dictionary label --> label_id
self.dict_label_id_ = {}
self.labels_ = []
for i, desc in enumerate(self.net_.label_names):
self.dict_label_id_[dict_desc_label[desc]] = i
self.labels_.append(dict_desc_label[desc])
# Load the mean vector from file
# mean of 3 channels
self.net_.mean_img =np.mean(np.mean(self.net_._data_mean,axis=1),axis=0)
def main():
net = DecafNet()
video = cv2.VideoCapture(0)
cascade = cv2.CascadeClassifier('haarcascade_frontalface_alt2.xml')
arrays = Queue.LifoQueue()
results = Queue.LifoQueue()
detector = ComputeFeatures(net, [], arrays, results)
detector.daemon = True
detector.start()
pygame.init()
screen = pygame.display.set_mode((width, height))
pygame.display.set_caption('This is a video game')
background = pygame.Surface((width, height))
background.fill(white)
screen.blit(background, (0, 0))
pygame.display.flip()
allsprites = pygame.sprite.RenderUpdates()
# Some parameters #
size = 10
enemy_surface = pygame.Surface((size, size))
speed = 200.0
playersize = 44
# # # # # # # # # #
player = Unit([256.0, 256.0], pygame.Surface((playersize, playersize)))
allsprites.add(player)
enemy_counter = 1.0
clock = pygame.time.Clock()
elapsed = 0.0
accumulator = 0.0
run = True
face = None
emotion_window = [
"neutral", "neutral", "neutral", "neutral", "neutral", "neutral"
]
#emotion_accumulator = 0.0
current_emotion = "neutral"
emotion = "neutral"
health = 50
game_time = 0.0
while run:
seconds = elapsed / 1000.0
accumulator += seconds
game_time += seconds
#emotion_accumulator += seconds
for event in pygame.event.get():
if event.type == pygame.QUIT:
run = False
elif event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE:
run = False
#elif event.type == pygame.KEYDOWN and event.key == pygame.K_w:
# current = Unit((random.randint(0,512), random.randint(0,512)))
# allsprites.add(current)
#elif event.type == pygame.KEYDOWN and event.key == pygame.K_s:
# for sprite in allsprites:
# sprite.position = [sprite.position[0]+random.randint(-5,5),sprite.position[1]+random.randint(-5,5)]
if accumulator > enemy_counter:
allsprites.add(Unit([random.randint(0, 512), 0], enemy_surface))
accumulator = 0.0
for sprite in allsprites.sprites():
if sprite.image == enemy_surface:
sprite.position[1] += speed * seconds
if sprite.position[1] > height - 10:
allsprites.remove(sprite)
pressed = pygame.key.get_pressed()
if pressed[pygame.K_RIGHT]:
player.position[0] += speed * seconds
if pressed[pygame.K_LEFT]:
player.position[0] -= speed * seconds
if pressed[pygame.K_DOWN]:
player.position[1] += speed * seconds
if pressed[pygame.K_UP]:
player.position[1] -= speed * seconds
allsprites.update()
allsprites.remove(player)
health -= len(pygame.sprite.spritecollide(player, allsprites, True))
allsprites.add(player)
allsprites.clear(screen, background)
changed = allsprites.draw(screen)
pygame.display.update(changed)
frame = video.read()[1]
rects = cascade.detectMultiScale(frame, 1.3, 3,
cv2.cv.CV_HAAR_SCALE_IMAGE,
(150, 150))
#arrays.put(frame)
# Idea: increase the size of the rectangle
if len(rects) > 0:
facerect = rects[0]
#facerect[0] -= (rectangle_margin-30)
#facerect[2] += rectangle_margin
#facerect[1] -= (rectangle_margin-20)
#facerect[3] += rectangle_margin
face = frame[facerect[1]:facerect[1] + facerect[3],
facerect[0]:facerect[0] + facerect[2]]
face = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)
arrays.put(face)
if True:
for (x, y, w, h) in rects:
cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0),
2)
if not results.empty():
emotion = results.get()
emotion_window.append(emotion)
emotion_window.pop(0)
current_emotion = max(set(emotion_window),
key=emotion_window.count)
print "Current emotion:", current_emotion, "- Last detected:", emotion #, emotion_window
if current_emotion == "happy":
enemy_counter += 0.03
enemy_counter = min(0.7, enemy_counter)
else:
enemy_counter += -0.02
enemy_counter = max(0.01, enemy_counter)
print "Health:", health, "- Time:", game_time
if health < 1:
run = False
print "Game over! Score:", game_time
if face != None:
cv2.imshow("face", face)
cv2.imshow("frame", frame)
c = cv2.waitKey(1)
if c == 27:
cv2.destroyWindow("frame")
cv2.destroyWindow("face")
break
elapsed = clock.tick(framerate)
video.release()
cv2.destroyAllWindows()
pygame.quit()
np.save(file_name + '_X.npy', features)
np.save(file_name + '_Y.npy', labels)
def load_samples(loc_dir="samples"):
logging.info("Loading samples from directory (samples)")
samples = []
for root, dirs, files in os.walk(loc_dir):
for f in files:
print f
if f[0:8] == "points99":
samples.append(("samples/s_negtive/%s"%f, 1))
elif f[0:8] == "points00":
samples.append(("samples/s_postive/%s"%f, 2))
else:
pass
return samples
if __name__ == "__main__":
"""decafnet."""
from decaf.scripts.imagenet import DecafNet
logging.getLogger().setLevel(logging.INFO)
net = DecafNet()
samples_list = load_samples()
#extractFeatures(samples_list, net, 'fc6_cudanet_out')
#writeFeatures_labels('420_decaf/420_decaf')
visualization()
joblib.dump({"420_decaf":FEATURES},"420_decaf/420pkl/420.pkl",compress=0) # ѹËõµ¼ÖÂÄÚ´æ²»×ã
#import cv2
#print 'prediction:', net.top_k_prediction(scores, 5)
#visualize.draw_net_to_file(net._net, 'decafnet.png')
#print 'Network structure written to decafnet.png'
class NetworkDecaf(Network):
"""
Implementation for the Decaf library.
"""
def __init__(self, model_spec_filename, model_filename=None,\
wnid_words_filename=None, center_only=False, wnid_subset = []):
"""
*** PRIVATE CONSTRUCTOR ***
"""
# the following is just an hack to allow retro-compatibility
# with existing code
if isinstance(model_spec_filename, NetworkDecafParams):
params = model_spec_filename
model_spec_filename = params.model_spec_filename
model_filename = params.model_filename
wnid_words_filename = params.wnid_words_filename
center_only = params.center_only
wnid_subset = params.wnid_subset
if wnid_subset!=[]:
print 'Warning: subset of labels not supported yet'
else:
assert isinstance(model_spec_filename, str)
assert model_filename != None
assert wnid_words_filename != None
# load Decaf model
self.net_ = DecafNet(model_filename, model_spec_filename)
self.center_only_ = center_only
# build a dictionary label --> description
self.dict_label_desc_ = {}
dict_desc_label = {}
fd = open(wnid_words_filename)
for line in fd:
temp = line.strip().split('\t')
wnid = temp[1].strip()
self.dict_label_desc_[wnid] = temp[2].strip()
dict_desc_label[temp[2].split(',')[0]] = wnid
fd.close()
# build a dictionary label --> label_id
self.dict_label_id_ = {}
self.labels_ = []
for i, desc in enumerate(self.net_.label_names):
self.dict_label_id_[dict_desc_label[desc]] = i
self.labels_.append(dict_desc_label[desc])
# Load the mean vector from file
# mean of 3 channels
self.net_.mean_img =np.mean(np.mean(self.net_._data_mean,axis=1),axis=0)
# it is in BGR convert in RGB
#self.net_.mean_img = self.net_.mean_img[::-1]
def get_mean_img(self):
return self.net_.mean_img
def get_input_dim(self):
return decaf.scripts.imagenet.INPUT_DIM
def get_label_id(self, label):
return self.dict_label_id_[label]
def get_label_desc(self, label):
return self.dict_label_desc_[label]
def get_labels(self):
return self.labels_
def evaluate(self, img, layer_name = 'softmax'):
# for now only center_only is supported
assert self.center_only_ == True
# first, extract the 227x227 center
dim = decaf.scripts.imagenet.INPUT_DIM
image = util.crop_image_center(decaf.util.transform.as_rgb(img))
image = skimage.transform.resize(image, (dim, dim))
# convert to [0,255] float32
image = image.astype(np.float32) * 255.
assert np.max(image) <= 255
# Flip the image if necessary, maintaining the c_contiguous order
if decaf.scripts.imagenet._JEFFNET_FLIP:
image = image[::-1, :].copy()
# subtract the mean, cropping the 256x256 mean image
xoff = (self.net_._data_mean.shape[1] - dim)/2
yoff = (self.net_._data_mean.shape[0] - dim)/2
image -= self.net_._data_mean[yoff+yoff+dim, xoff:xoff+dim]
# make sure the data in contiguous in memory
images = np.ascontiguousarray(image[np.newaxis], dtype=np.float32)
# classify
predictions = self.net_.classify_direct(images)
scores = predictions.mean(0)
# look at the particular layer
if layer_name == 'softmax':
return scores
elif layer_name == 'fc7_relu':
layer_name = 'fc7_neuron_cudanet_out'
elif layer_name == 'fc7':
layer_name = 'fc7_cudanet_out'
elif layer_name == 'fc6_relu':
layer_name = 'fc6_neuron_cudanet_out'
elif layer_name == 'fc6':
layer_name = 'fc6_cudanet_out'
elif layer_name == 'pool5':
layer_name = 'pool5_cudanet_out'
else:
raise ValueError('layer_name not supported')
return self.net_.feature(layer_name)
from classify.ttypes import *
# Thrift files
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
#*******************************************
import sys
sys.path.append("decaf")
from decaf.scripts.imagenet import DecafNet
import cStringIO as StringIO
import Image
import numpy as np
net = DecafNet('imagenet.decafnet.epoch90', 'imagenet.decafnet.meta')
from time import time as tic
#********************************************
# Server implementation
class ClassifyHandler:
## return current time stamp
#def showCurrentTimestamp(self):
# timeStamp = time.time()
# return str(timeStamp)
## print something to string, wait 10 secs, than print something again
#def asynchronousJob(self):
# print 'Assume that this work takes 10 seconds'
#!/usr/bin/python
import sys
sys.path.append("decaf")
from decaf.scripts.imagenet import DecafNet
import cStringIO as StringIO
import Image
import numpy as np
from time import time as tic
net = DecafNet('imagenet.decafnet.epoch90', 'imagenet.decafnet.meta')
start_time = tic()
#img = np.asarray(Image.open("cat.jpg"))
img_content = open("cat.jpg").read()
print len(img_content)
stream = StringIO.StringIO(img_content)
img = np.asarray(Image.open(stream))
scores = net.classify(img)
print net.top_k_prediction(scores, 5)
#scores = net.classify(img, center_only=True)
end_time = tic()
print "diff_time: %f" % (end_time - start_time)
from sklearn.base import TransformerMixin
from skimage.io import imread
import numpy as np
#import operator
import logging
logging.getLogger().setLevel(logging.ERROR)
from PIL import Image
img_size = (256,256,3)
def resize(img):
tmp = Image.fromarray(img)
tmp = tmp.resize(img_size[0:2])
return np.array(tmp)
from decaf.scripts.imagenet import DecafNet
NET = DecafNet()
class DecafFeature(TransformerMixin):
"""
Extract Decaf Feature
Parameters
----------
layer_name : str
Decaf layer name, default:fc6_cudanet_out
img_size : tuple
the size of X, default: (256, 256, 3)
"""
def __init__(self, layer='fc6_cudanet_out', img_size=(256, 256, 3)):
class DecafExtractor:
def __init__(self, layer_name,
model_path = 'dist/decaf-release/model/imagenet.decafnet.epoch90',
meta_path = 'dist/decaf-release/model/imagenet.decafnet.meta'):
self.layer_name = layer_name
self.net = DecafNet(model_path, meta_path)
self.transforms = [NopTransform()]
def set_parameters(self, patch_size, patches_per_image, levels, image_dim, decaf_oversample=False):
self.patch_size = patch_size
self.patches_per_image = patches_per_image
self.levels = levels
self.image_dim = image_dim
self.decaf_oversample = decaf_oversample
self.patch_sizes = map(int, self.patch_size * 2**np.arange(0,levels,1.0))
def add_transform(self, transform):
self.transforms.append(transform)
def get_descriptor_size(self):
if self.layer_name in ['67_relu', '67']:
return 8192
else:
return 4096
def get_decaf(self, im):
scores = self.net.classify(np.asarray(im), center_only=not self.decaf_oversample)
if self.layer_name == '67_relu':
return np.hstack([self.net.feature('fc6_neuron_cudanet_out'),
self.net.feature('fc7_neuron_cudanet_out')])
elif self.layer_name == '67':
return np.hstack([self.net.feature('fc6_cudanet_out'),
self.net.feature('fc7_cudanet_out')])
else:
return self.net.feature(self.layer_name)
def get_number_of_features_per_image(self):
if self.decaf_oversample:
return 10*len(self.transforms)
else:
return len(self.transforms)
def extract_image(self, filename):
""" This method extracts 4096-dimensional DeCAF features from
patches at multiple scales belonging to the the image file <filename>.
Number of scales, patch size, and other settings are set by method
set_parameters().
This method returns tuple (patches, positions, patch_number),
where <patches> is a numpy array of dimension (patch_number, 4096) holding features,
<positions> is a numpy array of dimension (patch_number, 2) holding absolute positions of patches,
<patch_number> is the number of patches that algorithm managed to extract -- it
is guaranteed to be at most the number of originally specified.
"""
log = get_logger()
im = Image.open(filename)
# Resizing image
if max(im.size) != self.image_dim:
if im.size[0] > im.size[1]:
new_height = (self.image_dim * im.size[1]) / im.size[0]
new_dim = (self.image_dim, new_height)
else:
new_width = (self.image_dim * im.size[0]) / im.size[1]
new_dim = (new_width, self.image_dim)
log.info('Resizing image from (%d, %d) to (%d, %d).', im.size[0], im.size[1], new_dim[0], new_dim[1])
im = im.resize(new_dim, Image.ANTIALIAS)
# Estimating number of extracted features taking into account transformations
estimated_feature_num = self.patches_per_image * self.get_number_of_features_per_image()
# Provisioning space for patches and locations
feature_storage = ExtractedFeatures(estimated_feature_num, self.get_descriptor_size())
log.info('Extracting up to %d patches at %d levels from "%s"...',
self.patches_per_image * self.get_number_of_features_per_image(),
self.levels, basename(filename))
# Applying transformations and extracting features
for xform in self.transforms:
im_ = xform.apply(im)
self.extract(im_, feature_storage, xform.check_coords, xform, filename)
log.info('Done. Extracted: %d.', feature_storage.cursor)
return feature_storage
def extract(self, im, feature_storage, check_patch_coords, transform, filename):
(w, h) = im.size
# Calculating patch step
if self.levels > 0:
patch_step = int( (w*h * len(self.patch_sizes) / self.patches_per_image)**0.5 )
w_steps = np.arange(0, w, patch_step)
h_steps = np.arange(0, h, patch_step)
(xx, yy) = np.meshgrid(w_steps, h_steps)
if isinstance(transform, NopTransform): # Hacky....
# Extracting features for the whole image
feature_storage.append( self.get_decaf(im), np.matrix([0,0]) )
# Extracting features from patches
for l in range(self.levels):
for i in range(xx.shape[0]):
for j in range(xx.shape[1]):
x = xx[i,j]
y = yy[i,j]
patch_left = x+self.patch_sizes[l]
patch_bottom = y+self.patch_sizes[l]
if (check_patch_coords(x, y, patch_left, patch_bottom) and
patch_left <= w and patch_bottom <= h ):
patch = im.crop( (x, y, patch_left, patch_bottom) )
patch.load()
feature_storage.append( self.get_decaf(patch), np.matrix([x, y]) )
class ConvNetFeatures(BaseEstimator):
"""Extract features from images using a pretrained ConvNet.
Based on Yangqing Jia and Jeff Donahue's `DeCAF
<https://github.com/UCB-ICSI-Vision-Group/decaf-release/wiki>`_.
Please make sure you read and accept DeCAF's license before you
use this class.
If ``classify_direct=False``, expects its input X to be a list of
image filenames or arrays as produced by
`np.array(Image.open(filename))`.
"""
verbose = 0
def __init__(
self,
feature_layer='fc7_cudanet_out',
pretrained_params='imagenet.decafnet.epoch90',
pretrained_meta='imagenet.decafnet.meta',
center_only=True,
classify_direct=False,
verbose=0,
):
"""
:param feature_layer: The ConvNet layer that's used for
feature extraction. Defaults to
`fc7_cudanet_out`. A description of all
available layers for the
ImageNet-1k-pretrained ConvNet is found
in the DeCAF wiki. They are:
- `pool5_cudanet_out`
- `fc6_cudanet_out`
- `fc6_neuron_cudanet_out`
- `fc7_cudanet_out`
- `fc7_neuron_cudanet_out`
- `probs_cudanet_out`
:param pretrained_params: This must point to the file with the
pretrained parameters. Defaults to
`imagenet.decafnet.epoch90`. For
the ImageNet-1k-pretrained ConvNet
this file can be obtained from here:
http://www.eecs.berkeley.edu/~jiayq/decaf_pretrained/
:param pretrained_meta: Similar to `pretrained_params`, this
must file to the file with the
pretrained parameters' metadata.
Defaults to `imagenet.decafnet.meta`.
:param center_only: Use the center patch of the image only
when extracting features. If `False`, use
four corners, the image center and flipped
variants and average a total of 10 feature
vectors, which will usually yield better
results. Defaults to `True`.
:param classify_direct: When `True`, assume that input X is an
array of shape (num x 256 x 256 x 3)
as returned by `prepare_image`.
"""
self.feature_layer = feature_layer
self.pretrained_params = pretrained_params
self.pretrained_meta = pretrained_meta
self.center_only = center_only
self.classify_direct = classify_direct
self.net_ = None
if (not os.path.exists(pretrained_params) or
not os.path.exists(pretrained_meta)):
raise ValueError(
"Pre-trained ConvNet parameters not found. You may"
"need to download the files from "
"http://www.eecs.berkeley.edu/~jiayq/decaf_pretrained/ and "
"pass the path to the two files as `pretrained_params` and "
"`pretrained_meta` to the `{}` estimator.".format(
self.__class__.__name__))
def fit(self, X=None, y=None):
from decaf.scripts.imagenet import DecafNet # soft dep
if self.net_ is None:
self.net_ = DecafNet(
self.pretrained_params,
self.pretrained_meta,
)
return self
@cache.cached(_transform_cache_key)
def transform(self, X):
features = []
for img in X:
if self.classify_direct:
images = self.net_.oversample(
img, center_only=self.center_only)
self.net_.classify_direct(images)
else:
if isinstance(img, str):
import Image # soft dep
img = np.array(Image.open(img))
self.net_.classify(img, center_only=self.center_only)
feat = None
for layer in self.feature_layer.split(','):
val = self.net_.feature(layer)
if feat is None:
feat = val
else:
feat = np.hstack([feat, val])
if not self.center_only:
feat = feat.flatten()
features.append(feat)
if self.verbose:
sys.stdout.write(
"\r[ConvNet] %d%%" % (100. * len(features) / len(X)))
sys.stdout.flush()
if self.verbose:
sys.stdout.write('\n')
return np.vstack(features)
def prepare_image(self, image):
"""Returns image of shape `(256, 256, 3)`, as expected by
`transform` when `classify_direct = True`.
"""
from decaf.util import transform # soft dep
_JEFFNET_FLIP = True
# first, extract the 256x256 center.
image = transform.scale_and_extract(transform.as_rgb(image), 256)
# convert to [0,255] float32
image = image.astype(np.float32) * 255.
if _JEFFNET_FLIP:
# Flip the image if necessary, maintaining the c_contiguous order
image = image[::-1, :].copy()
# subtract the mean
image -= self.net_._data_mean
return image
new_shape = (int(k * w), int(k * h))
resized = transform.resize(readed, new_shape)
else:
resized = readed
return resized, np.asarray(resized)
if __name__ == '__main__':
folder = sys.argv[1]
out_folder = sys.argv[2]
if os.path.exists(out_folder):
shutil.rmtree(out_folder)
os.mkdir(out_folder)
imgs = glob.glob(os.path.join(folder,"*.jpeg"))
print imgs
net = DecafNet('../imagenet_pretrained/imagenet.decafnet.epoch90', '../imagenet_pretrained/imagenet.decafnet.meta')
flog = open('log.txt', 'w')
for i, imgname in enumerate(imgs):
flog.write("%s\t%d" % (imgname, i))
try:
resized, img = load_and_resize(imgname)
except ValueError:
print "error when read %s" % imgname
continue
scores = net.classify(img, center_only=True)
feature = net.feature('fc6_cudanet_out')
print feature
out_file = open(os.path.join(out_folder, "%d.npy" % i), 'w')
def __init__(self, layer_name,
model_path = 'dist/decaf-release/model/imagenet.decafnet.epoch90',
meta_path = 'dist/decaf-release/model/imagenet.decafnet.meta'):
self.layer_name = layer_name
self.net = DecafNet(model_path, meta_path)
self.transforms = [NopTransform()]
from decaf.scripts.imagenet import DecafNet
from skimage import io
import numpy as np
import scipy.io as sio
import os
import sys
if len(sys.argv) != 5:
print "Usage ", sys.argv[0], "<model_root_dir> <image_dir> <output_feature_path> <num_imgs>"
exit(1)
model_root = sys.argv[1]
net = DecafNet(model_root + 'imagenet.decafnet.epoch90', model_root + 'imagenet.decafnet.meta')
img_dir = sys.argv[2]
feature_path = sys.argv[3]
NUM_IMGS = int(sys.argv[4])
FEATURE_DIM = 4096 #fc6_cudanet_out's dimension
features = np.zeros((NUM_IMGS,FEATURE_DIM))
for i in range(NUM_IMGS):
filename = img_dir + "/%05d.jpg" %(i+1)
if os.path.exists(filename):
sys.stdout.write("Extracting DeCAF feature from image %d\n" %(i+1))
img = io.imread(filename)
net.classify(img, center_only=True)
features[i,:] = net.feature('fc6_cudanet_out')
sio.savemat(feature_path,{'features':features})
try:
return img_as_ubyte(resize(img, (256,256), mode='wrap')) # resize 后是float64
except:
#保存检测过程图像
io.imsave("420_decaf/tmp/%s_%s_%s_%s.png" % \
(lu_offset_x, lu_offset_y, w, h), img)
tmp = cv2.imread("420_decaf/tmp/%s_%s_%s_%s.png" % \
(lu_offset_x, lu_offset_y, w, h))
return cv2.resize(img, (256,256), interpolation=cv2.INTER_LINEAR)
#return resize(img, (256,256))
# 加载 decaf 和 classifier
from decaf.scripts.imagenet import DecafNet
net = DecafNet()
clf = joblib.load("420_decaf/classifier_svc.pkl")
blob_name='fc6_cudanet_out'
# 命令行参数情况
if len(sys.argv) < 3:
print "usage: object_classify.py path_to_image path_to_segmentation_folder..."
sys.exit()
else:
img_path = sys.argv[1]
segmentation_folder = sys.argv[2:]
# 读取栅格图像
g_raster = gdal.Open(img_path) # 与分割文件对应的原始栅格
# 读取分割结果 shp 文件
class NetworkDecaf(Network):
"""
Implementation for the Decaf library.
"""
def __init__(self, model_spec_filename, model_filename=None,\
wnid_words_filename=None, center_only=False, wnid_subset = []):
"""
*** PRIVATE CONSTRUCTOR ***
"""
# the following is just an hack to allow retro-compatibility
# with existing code
if isinstance(model_spec_filename, NetworkDecafParams):
params = model_spec_filename
model_spec_filename = params.model_spec_filename
model_filename = params.model_filename
wnid_words_filename = params.wnid_words_filename
center_only = params.center_only
wnid_subset = params.wnid_subset
if wnid_subset != []:
print 'Warning: subset of labels not supported yet'
else:
assert isinstance(model_spec_filename, str)
assert model_filename != None
assert wnid_words_filename != None
# load Decaf model
self.net_ = DecafNet(model_filename, model_spec_filename)
self.center_only_ = center_only
# build a dictionary label --> description
self.dict_label_desc_ = {}
dict_desc_label = {}
fd = open(wnid_words_filename)
for line in fd:
temp = line.strip().split('\t')
wnid = temp[1].strip()
self.dict_label_desc_[wnid] = temp[2].strip()
dict_desc_label[temp[2].split(',')[0]] = wnid
fd.close()
# build a dictionary label --> label_id
self.dict_label_id_ = {}
self.labels_ = []
for i, desc in enumerate(self.net_.label_names):
self.dict_label_id_[dict_desc_label[desc]] = i
self.labels_.append(dict_desc_label[desc])
# Load the mean vector from file
# mean of 3 channels
self.net_.mean_img = np.mean(np.mean(self.net_._data_mean, axis=1),
axis=0)
# it is in BGR convert in RGB
#self.net_.mean_img = self.net_.mean_img[::-1]
def get_mean_img(self):
return self.net_.mean_img
def get_input_dim(self):
return decaf.scripts.imagenet.INPUT_DIM
def get_label_id(self, label):
return self.dict_label_id_[label]
def get_label_desc(self, label):
return self.dict_label_desc_[label]
def get_labels(self):
return self.labels_
def evaluate(self, img, layer_name='softmax'):
# for now only center_only is supported
assert self.center_only_ == True
# first, extract the 227x227 center
dim = decaf.scripts.imagenet.INPUT_DIM
image = util.crop_image_center(decaf.util.transform.as_rgb(img))
image = skimage.transform.resize(image, (dim, dim))
# convert to [0,255] float32
image = image.astype(np.float32) * 255.
assert np.max(image) <= 255
# Flip the image if necessary, maintaining the c_contiguous order
if decaf.scripts.imagenet._JEFFNET_FLIP:
image = image[::-1, :].copy()
# subtract the mean, cropping the 256x256 mean image
xoff = (self.net_._data_mean.shape[1] - dim) / 2
yoff = (self.net_._data_mean.shape[0] - dim) / 2
image -= self.net_._data_mean[yoff + yoff + dim, xoff:xoff + dim]
# make sure the data in contiguous in memory
images = np.ascontiguousarray(image[np.newaxis], dtype=np.float32)
# classify
predictions = self.net_.classify_direct(images)
scores = predictions.mean(0)
# look at the particular layer
if layer_name == 'softmax':
return scores
elif layer_name == 'fc7_relu':
layer_name = 'fc7_neuron_cudanet_out'
elif layer_name == 'fc7':
layer_name = 'fc7_cudanet_out'
elif layer_name == 'fc6_relu':
layer_name = 'fc6_neuron_cudanet_out'
elif layer_name == 'fc6':
layer_name = 'fc6_cudanet_out'
elif layer_name == 'pool5':
layer_name = 'pool5_cudanet_out'
else:
raise ValueError('layer_name not supported')
return self.net_.feature(layer_name)
from decaf.scripts.imagenet import DecafNet
import numpy, scipy, PIL, csv, glob
import numpy as np
from PIL import Image
from sklearn.decomposition import PCA
import os
import cv2, scipy
import pickle
from mlabwrap import mlab
ucfloc = 'decaf/KitchenData/'
imgnetPath = 'decaf/imagenet_pretrained/'
flowdir = 'flowdata/'
net = DecafNet(imgnetPath + 'imagenet.decafnet.epoch90',
imgnetPath + 'imagenet.decafnet.meta')
pca = PCA(n_components=20)
class Feature():
def __init__(self, decaf, category, _id):
self.decaf = decaf
self.category = category
self.path = _id
def imToNumpy(img):
return numpy.asarray(PIL.Image.open(img))
def getFeature(img):
class DecafExtractor:
def __init__(
self,
layer_name,
model_path='dist/decaf-release/model/imagenet.decafnet.epoch90',
meta_path='dist/decaf-release/model/imagenet.decafnet.meta'):
self.layer_name = layer_name
self.net = DecafNet(model_path, meta_path)
self.transforms = [NopTransform()]
def set_parameters(self,
patch_size,
patches_per_image,
levels,
image_dim,
decaf_oversample=False):
self.patch_size = patch_size
self.patches_per_image = patches_per_image
self.levels = levels
self.image_dim = image_dim
self.decaf_oversample = decaf_oversample
self.patch_sizes = map(int,
self.patch_size * 2**np.arange(0, levels, 1.0))
def add_transform(self, transform):
self.transforms.append(transform)
def get_descriptor_size(self):
if self.layer_name in ['67_relu', '67']:
return 8192
else:
return 4096
def get_decaf(self, im):
scores = self.net.classify(np.asarray(im),
center_only=not self.decaf_oversample)
if self.layer_name == '67_relu':
return np.hstack([
self.net.feature('fc6_neuron_cudanet_out'),
self.net.feature('fc7_neuron_cudanet_out')
])
elif self.layer_name == '67':
return np.hstack([
self.net.feature('fc6_cudanet_out'),
self.net.feature('fc7_cudanet_out')
])
else:
return self.net.feature(self.layer_name)
def get_number_of_features_per_image(self):
if self.decaf_oversample:
return 10 * len(self.transforms)
else:
return len(self.transforms)
def extract_image(self, filename):
""" This method extracts 4096-dimensional DeCAF features from
patches at multiple scales belonging to the the image file <filename>.
Number of scales, patch size, and other settings are set by method
set_parameters().
This method returns tuple (patches, positions, patch_number),
where <patches> is a numpy array of dimension (patch_number, 4096) holding features,
<positions> is a numpy array of dimension (patch_number, 2) holding absolute positions of patches,
<patch_number> is the number of patches that algorithm managed to extract -- it
is guaranteed to be at most the number of originally specified.
"""
log = get_logger()
im = Image.open(filename)
# Resizing image
if max(im.size) != self.image_dim:
if im.size[0] > im.size[1]:
new_height = (self.image_dim * im.size[1]) / im.size[0]
new_dim = (self.image_dim, new_height)
else:
new_width = (self.image_dim * im.size[0]) / im.size[1]
new_dim = (new_width, self.image_dim)
log.info('Resizing image from (%d, %d) to (%d, %d).', im.size[0],
im.size[1], new_dim[0], new_dim[1])
im = im.resize(new_dim, Image.ANTIALIAS)
# Estimating number of extracted features taking into account transformations
estimated_feature_num = self.patches_per_image * self.get_number_of_features_per_image(
)
# Provisioning space for patches and locations
feature_storage = ExtractedFeatures(estimated_feature_num,
self.get_descriptor_size())
log.info(
'Extracting up to %d patches at %d levels from "%s"...',
self.patches_per_image * self.get_number_of_features_per_image(),
self.levels, basename(filename))
# Applying transformations and extracting features
for xform in self.transforms:
im_ = xform.apply(im)
self.extract(im_, feature_storage, xform.check_coords, xform,
filename)
log.info('Done. Extracted: %d.', feature_storage.cursor)
return feature_storage
def extract(self, im, feature_storage, check_patch_coords, transform,
filename):
(w, h) = im.size
# Calculating patch step
if self.levels > 0:
patch_step = int(
(w * h * len(self.patch_sizes) / self.patches_per_image)**0.5)
w_steps = np.arange(0, w, patch_step)
h_steps = np.arange(0, h, patch_step)
(xx, yy) = np.meshgrid(w_steps, h_steps)
if isinstance(transform, NopTransform): # Hacky....
# Extracting features for the whole image
feature_storage.append(self.get_decaf(im), np.matrix([0, 0]))
# Extracting features from patches
for l in range(self.levels):
for i in range(xx.shape[0]):
for j in range(xx.shape[1]):
x = xx[i, j]
y = yy[i, j]
patch_left = x + self.patch_sizes[l]
patch_bottom = y + self.patch_sizes[l]
if (check_patch_coords(x, y, patch_left, patch_bottom)
and patch_left <= w and patch_bottom <= h):
patch = im.crop((x, y, patch_left, patch_bottom))
patch.load()
feature_storage.append(self.get_decaf(patch),
np.matrix([x, y]))
