def eval_video(gpu_id): global trunk_net_base print 'GPU %d is processing' % gpu_id score_name_list = ['fc', 'fc_motion', 'fc_motion_14'] # initialization net = CaffeNet('{}/deploy_{}.prototxt'.format(trunk_net_base, gpu_id), args.net_weights, gpu_id) # preparing video names video_names = [] with open('{}/source_{}'.format(trunk_net_base, gpu_id)) as fsource_split: video_names = map(lambda l: os.path.basename(l.split(' ')[0]), fsource_split.readlines()) length_split = len(video_names) video_result = [] for video_id in xrange(length_split): with open('{}/video_id_{}'.format(trunk_net_base, gpu_id), 'w') as fvid: fvid.write('{}'.format(video_id)) scores, label = net.predict_single_frame_motion(trunk_net_base, score_name_list, gpu_id, over_sample=True) print '[Worker {}] video {}: {} Done'.format(gpu_id, video_id, video_names[video_id]) video_result.append((scores, label, video_names[video_id])) return video_result
def build_net(): global net my_id = multiprocessing.current_process()._identity[0] \ if args.num_worker > 1 else 1 if gpu_list is None: net = CaffeNet(args.net_proto, args.net_weights, my_id-1) else: net = CaffeNet(args.net_proto, args.net_weights, gpu_list[my_id - 1])
def reconfig_srv(self, req): self.image_sub.unregister() self.split = req.Split rospy.loginfo("reading split:"+str(req.Split)) #print(req.Split) self.caffemodel = mypath+'/models/'+ self.dataset +'_split_'+str(self.split)+'_tsn_rgb_reference_bn_inception.caffemodel' self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.image_sub = rospy.Subscriber(self.videotopic, Image,self.callback,queue_size=1) #print('Dum') return []
def build_net(): global net my_id = multiprocessing.current_process()._identity[0] \ if num_worker > 1 else 1 print "my", my_id if num_worker == 1: net = CaffeNet(net_proto, net_weights, num_id) else: if gpu_list is None: net = CaffeNet(net_proto, net_weights, my_id - 1) else: net = CaffeNet(net_proto, net_weights, gpu_list[my_id - 1])
def build_net(): net_proto = '/home/gzn/code/repository/temporal-segment-networks/models/bn_inception_kinetics_rgb_pretrained/bn_inception_rgb_deploy.prototxt' net_weights = '/home/gzn/code/repository/temporal-segment-networks/models/bn_inception_kinetics_rgb_pretrained/bn_inception_kinetics_rgb_pretrained.caffemodel' num_worker = 2 gpu_list = [0, 1] global net my_id = multiprocessing.current_process()._identity[0] \ if num_worker > 1 else 1 if gpu_list is None: net = CaffeNet(net_proto, net_weights, my_id - 1) else: net = CaffeNet(net_proto, net_weights, gpu_list[my_id - 1])
def build_net(net_proto, net_weights, max_gpu_id=10): # Note: CaffeNet creates a caffe.TEST net, so dropout layer does not # drop anything and the TEST input data layer, if present, is used. global net my_id = multiprocessing.current_process()._identity[0] \ if args.num_worker > 1 else 1 if gpu_list is None: assert (my_id <= max_gpu_id) net = CaffeNet(net_proto, net_weights, my_id - 1) else: gpu_id = (my_id - 1) % len(gpu_list) net = CaffeNet(net_proto, net_weights, gpu_list[gpu_id]) return
def build_net(): net_proto = '/home/gzn/code/repository/temporal-segment-networks/models/bn_inception_kinetics_flow_pretrained/bn_inception_flow_deploy.prototxt' net_weights = '/home/gzn/code/repository/temporal-segment-networks/models/bn_inception_kinetics_flow_pretrained/bn_inception_kinetics_flow_pretrained.caffemodel' num_worker = 4 gpu_list = [0, 1, 2, 3] global net global denFlow my_id = multiprocessing.current_process()._identity[0] \ if num_worker > 1 else 1 if gpu_list is None: net = CaffeNet(net_proto, net_weights, my_id - 1) denFlow = DenFlow.TVL1FlowExtractor(20) denFlow.set_device(my_id - 1) else: net = CaffeNet(net_proto, net_weights, gpu_list[my_id - 1]) denFlow = DenFlow.TVL1FlowExtractor(20) denFlow.set_device(gpu_list[my_id - 1])
def __init__(self): global mypath # services provided self.reconfig_srv_ = rospy.Service('reconf_split',split, self.reconfig_srv) self.start_vidscores = rospy.Service('start_vidscores', Empty, self.start_vidscores) self.stop_vidscores = rospy.Service('stop_vidscores', Empty, self.stop_vidscores) # topics published self.image_pub = rospy.Publisher("class_overlay_image_raw",Image, queue_size=1) self.label_fw_pub = rospy.Publisher("action_fw", String, queue_size=1) self.label_pub = rospy.Publisher("action", String, queue_size=1) self.ownlabel_pub = rospy.Publisher("action_own", String, queue_size=1) # parameters self.dataset = rospy.get_param('~dataset','hmdb51') self.device_id = rospy.get_param('~device_id',0) self.split = rospy.get_param('~split',1) self.videotopic = rospy.get_param('~video_topic','videofiles/image_raw') self.classwindow = rospy.get_param('~classification_frame_window',50) self.actionlist = rospy.get_param('~action_list', ['brush_hair','cartwheel','catch','chew','clap','climb','climb_stairs','dive','draw_sword','dribble','drink','eat','fall_floor','fencing','flic_flac','golf','handstand','hit','hug','jump','kick','kick_ball','kiss','laugh','pick','pour','pullup','punch','push','pushup','ride_bike','ride_horse','run','shake_hands','shoot_ball','shoot_bow','shoot_gun','sit','situp','smile','smoke','somersault','stand','swing_baseball','sword','sword_exercise','talk','throw','turn','walk','wave']) if type(self.actionlist) is str: self.actionlist = eval(self.actionlist) self.actionlist.sort() self.chooselist = rospy.get_param('~choose_list',[]) if type(self.chooselist) is str: self.chooselist = eval(self.chooselist) self.chooselist.sort() ###probably should use the nice rosparam thingy here to avoid these problems... self.framesize_width = rospy.get_param('~framesize_width',340) self.framesize_height = rospy.get_param('~framesize_height',256) # topics subscribed self.image_sub = rospy.Subscriber(self.videotopic, Image,self.callback,queue_size=1) # internals self.bridge = CvBridge() from pyActionRecog.utils.video_funcs import default_aggregation_func if self.chooselist: keepi = [] rospy.logwarn('defined own subset of actions! classification will be reduced to smaller set of choices, namely:'+str(self.chooselist)) #print(range(0,len(self.actionlist))) for i in range(0,len(self.actionlist)): for j in range(0, len(self.chooselist)): #print(self.actionlist[i]) #print( self.chooselist[j]) if self.actionlist[i] == self.chooselist[j]: keepi.append(i) tobedeleted = set(range(0,len(self.actionlist)))-set(keepi) #print(tobedeleted) self.defprox = lambda x: np.delete(default_aggregation_func(x),list(tobedeleted)) self.actionlist = self.chooselist else: rospy.logwarn('No choose_list defined. Will classify within the whole set. ') self.defprox = default_aggregation_func self.frame_scores = [] self.prototxt = mypath+'/models/'+ self.dataset +'/tsn_bn_inception_rgb_deploy.prototxt' self.caffemodel = mypath+'/models/'+ self.dataset +'_split_'+str(self.split)+'_tsn_rgb_reference_bn_inception.caffemodel' self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.font = cv2.FONT_HERSHEY_SIMPLEX #print('hio') self.ownvidscores = [] # when I instantiate the classifier, the startedownvid is working already. this influences how vsmf_srv will behave, so it needs to be like this, I think. self.startedownvid = True self.lock = threading.Lock() rospy.loginfo("waiting for callback from " + self.videotopic +" to do anything")
def build_net(): global net net = CaffeNet(args.net_proto, args.net_weights, 0)
sys.stderr.write('frame x name:', os.path.join(video_frame_path, x_name) + '\n') scores = np.zeros_like(frame_scores[0]) frame_scores.append(scores) print 'video {} done'.format(vid) sys.stdin.flush() return np.array(frame_scores), label if args.num_worker > 1: pool = multiprocessing.Pool(args.num_worker, initializer=build_net) video_scores = pool.map(eval_video, eval_video_list) else: # build_net() net = CaffeNet(args.net_proto, args.net_weights, 0) video_scores = map(eval_video, eval_video_list) video_pred = [np.argmax(default_aggregation_func(x[0])) for x in video_scores] video_labels = [x[1] for x in video_scores] cf = confusion_matrix(video_labels, video_pred).astype(float) cls_cnt = cf.sum(axis=1) cls_hit = np.diag(cf) cls_acc = cls_hit / cls_cnt print cls_acc print 'Accuracy {:.02f}%'.format(np.mean(cls_acc) * 100)
def build_net(gpu_id): # global net # gpu_id = gpu_list[current_process()._identity[0] % len(gpu_list)] # gpu_id = 1 net = CaffeNet(deploy_file, caffemodel, gpu_id) return net
def build_net(): global net my_id = multiprocessing.current_process()._identity[0] \ if args.num_worker > 1 else 1 net = CaffeNet(args.net_proto, args.net_weights, my_id - 1)
import os import cv2 import numpy as np from sklearn.metrics import confusion_matrix from pyActionRecog import parse_directory from pyActionRecog import parse_split_file from pyActionRecog.utils.video_funcs import default_aggregation_func from pyActionRecog.action_caffe import CaffeNet net = CaffeNet( './models/ixmas_branch/mp_in5b_crf_1/split1/rgb_deploy.prototxt', './models/ixmas_branch/mp_in5b_crf_1/caffemodel/rgb_split1_iter_7000.caffemodel', 0) split_tp = parse_split_file('ixmas_branch') f_info = parse_directory('../data/IXMAS/img_flow', 'img_', 'flow_x_', 'flow_y_') eval_video_list = split_tp[0][1] video = eval_video_list[0] vid = video[0] video_frame_path = f_info[0][vid] tick = 5 name = '{}{:05d}.jpg'.format('img_', tick) frame = cv2.imread(os.path.join(video_frame_path, name), cv2.IMREAD_COLOR)
def __init__(self): global mypath # services provided self.reconfig_srv_ = rospy.Service('reconf_split', split, self.reconfig_srv) self.start_vidscores = rospy.Service('start_vidscores', Empty, self.start_vidscores) self.stop_vidscores = rospy.Service('stop_vidscores', Empty, self.stop_vidscores) # topics published self.scores_pub = rospy.Publisher("scores", Float32MultiArray, queue_size=1) self.score_fw_pub = rospy.Publisher("scores_fw", caffe_tsn_ros.msg.ScoreArray, queue_size=1) # self.label_pub = rospy.Publisher("action", String, queue_size=1) self.ownlabel_pub = rospy.Publisher("scores_own", caffe_tsn_ros.msg.ScoreArray, queue_size=1) # parameters self.dataset = rospy.get_param('~dataset', 'hmdb51') self.device_id = rospy.get_param('~device_id', 0) self.split = rospy.get_param('~split', 1) self.rgbOrFlow = rospy.get_param('~classifier_type') self.clear_fw_after_new_vid = rospy.get_param( '~clear_fw_after_new_vid', True) self.step = rospy.get_param('~step', 6) # this should actually be # step = (frame_cnt - stack_depth) / (args.num_frame_per_video-1) # it will change depending on the action length, a value I don't have if I am classifying real time, but that I could get if I am doing it by service calls! # stack_depth is 1 for rgb and 5 for flows. I am letting it be 5 to test creating an array of cv_images self.stack_depth = rospy.get_param('~stack_depth', 5) self.stack_count = 0 self.cv_image_stack = [] self.classwindow = rospy.get_param('~classification_frame_window', 'undefined') assert not self.classwindow == 'undefined' ###probably should use the nice rosparam thingy here to avoid these problems... self.framesize_width = rospy.get_param('~framesize_width', 340) self.framesize_height = rospy.get_param('~framesize_height', 256) # topics subscribed self.image_sub = rospy.Subscriber('video_topic', Image, self.callback, queue_size=1) # internals self.bridge = CvBridge() self.prototxt = mypath + '/models/' + self.dataset + '/tsn_bn_inception_' + self.rgbOrFlow + '_deploy.prototxt' self.caffemodel = mypath + '/models/' + self.dataset + '_split_' + str( self.split ) + '_tsn_' + self.rgbOrFlow + '_reference_bn_inception.caffemodel' rospy.loginfo("loading prototxt {}".format(self.prototxt)) rospy.loginfo("loading caffemodel {}".format(self.caffemodel)) self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.frame_scores = caffe_tsn_ros.msg.ScoreArray() self.ownvidscores = caffe_tsn_ros.msg.ScoreArray() # when I instantiate the classifier, the startedownvid is working already. this influences how vsmf_srv will behave, so it needs to be like this, I think. self.startedownvid = True self.lock = threading.Lock() self.published_fw = False rospy.set_param('~alive', 0.5) rospy.loginfo("waiting for callback from " + rospy.resolve_name('video_topic') + " to do anything")
def build_net(): global net gpu_id = gpu_list[current_process()._identity[0] % len(gpu_list)] net = CaffeNet(deploy_file, caffemodel, gpu_id)
class tsn_classifier: def __init__(self): global mypath # services provided self.reconfig_srv_ = rospy.Service('reconf_split',split, self.reconfig_srv) self.start_vidscores = rospy.Service('start_vidscores', Empty, self.start_vidscores) self.stop_vidscores = rospy.Service('stop_vidscores', Empty, self.stop_vidscores) # topics published self.image_pub = rospy.Publisher("class_overlay_image_raw",Image, queue_size=1) self.label_fw_pub = rospy.Publisher("action_fw", String, queue_size=1) self.label_pub = rospy.Publisher("action", String, queue_size=1) self.ownlabel_pub = rospy.Publisher("action_own", String, queue_size=1) # parameters self.dataset = rospy.get_param('~dataset','hmdb51') self.device_id = rospy.get_param('~device_id',0) self.split = rospy.get_param('~split',1) self.videotopic = rospy.get_param('~video_topic','videofiles/image_raw') self.classwindow = rospy.get_param('~classification_frame_window',50) self.actionlist = rospy.get_param('~action_list', ['brush_hair','cartwheel','catch','chew','clap','climb','climb_stairs','dive','draw_sword','dribble','drink','eat','fall_floor','fencing','flic_flac','golf','handstand','hit','hug','jump','kick','kick_ball','kiss','laugh','pick','pour','pullup','punch','push','pushup','ride_bike','ride_horse','run','shake_hands','shoot_ball','shoot_bow','shoot_gun','sit','situp','smile','smoke','somersault','stand','swing_baseball','sword','sword_exercise','talk','throw','turn','walk','wave']) if type(self.actionlist) is str: self.actionlist = eval(self.actionlist) self.actionlist.sort() self.chooselist = rospy.get_param('~choose_list',[]) if type(self.chooselist) is str: self.chooselist = eval(self.chooselist) self.chooselist.sort() ###probably should use the nice rosparam thingy here to avoid these problems... self.framesize_width = rospy.get_param('~framesize_width',340) self.framesize_height = rospy.get_param('~framesize_height',256) # topics subscribed self.image_sub = rospy.Subscriber(self.videotopic, Image,self.callback,queue_size=1) # internals self.bridge = CvBridge() from pyActionRecog.utils.video_funcs import default_aggregation_func if self.chooselist: keepi = [] rospy.logwarn('defined own subset of actions! classification will be reduced to smaller set of choices, namely:'+str(self.chooselist)) #print(range(0,len(self.actionlist))) for i in range(0,len(self.actionlist)): for j in range(0, len(self.chooselist)): #print(self.actionlist[i]) #print( self.chooselist[j]) if self.actionlist[i] == self.chooselist[j]: keepi.append(i) tobedeleted = set(range(0,len(self.actionlist)))-set(keepi) #print(tobedeleted) self.defprox = lambda x: np.delete(default_aggregation_func(x),list(tobedeleted)) self.actionlist = self.chooselist else: rospy.logwarn('No choose_list defined. Will classify within the whole set. ') self.defprox = default_aggregation_func self.frame_scores = [] self.prototxt = mypath+'/models/'+ self.dataset +'/tsn_bn_inception_rgb_deploy.prototxt' self.caffemodel = mypath+'/models/'+ self.dataset +'_split_'+str(self.split)+'_tsn_rgb_reference_bn_inception.caffemodel' self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.font = cv2.FONT_HERSHEY_SIMPLEX #print('hio') self.ownvidscores = [] # when I instantiate the classifier, the startedownvid is working already. this influences how vsmf_srv will behave, so it needs to be like this, I think. self.startedownvid = True self.lock = threading.Lock() rospy.loginfo("waiting for callback from " + self.videotopic +" to do anything") def start_vidscores(self,req): # I will need to use locks here, I think... self.lock.acquire() self.startedownvid = True rospy.logwarn("Started classifying own vid!") self.lock.release() return [] def stop_vidscores(self,req): # I will need to use locks here, I think... self.lock.acquire() self.startedownvid = False if self.ownvidscores: currown = self.actionlist[np.argmax(self.defprox(self.ownvidscores))] self.ownlabel_pub.publish(String(currown)) else: rospy.logerr('ownvidscores is empty!!!!!!!!!!!!!!! are we locking for too long?') self.ownvidscores = [] rospy.logdebug("published the label for the own video version!") rospy.logwarn("stopped classifying own vid") self.lock.release() return [] def callback(self,data): rospy.logdebug("reached callback. that means I can read the Subscriber!") try: cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: print(e) #(rows,cols,channels) = cv_image.shape #if cols > 60 and rows > 60 : #cv2.circle(cv_image, (50,50), 10, 255) #TODO: frame_size should be a parameter scores = self.net.predict_single_frame([cv_image,], 'fc-action', frame_size=(self.framesize_width, self.framesize_height)) #print((scores)) #this publishes the instant time version, aka, per frame self.label_pub.publish(String(self.actionlist[np.argmax(self.defprox([scores]))])) rospy.logdebug("published the label for instant time version!") #this part publishes the frame_window version self.frame_scores.append(scores) if len(self.frame_scores)>self.classwindow: curract_fw = self.actionlist[np.argmax(self.defprox(self.frame_scores))] cv2.putText(cv_image,curract_fw,(10,226), self.font, 1,(255,255,255),2) self.frame_scores.pop(0) self.label_fw_pub.publish(String(curract_fw)) rospy.logdebug("published the label for the frame window version!") self.lock.acquire() if self.startedownvid: self.ownvidscores.append(scores) else: rospy.logdebug_throttle(20,"waiting for start_vidscores call to start classifying ownvid") self.lock.release() try: self.image_pub.publish(self.bridge.cv2_to_imgmsg(cv_image, "bgr8")) rospy.logdebug("published image") except CvBridgeError as e: print(e) def reconfig_srv(self, req): self.image_sub.unregister() self.split = req.Split rospy.loginfo("reading split:"+str(req.Split)) #print(req.Split) self.caffemodel = mypath+'/models/'+ self.dataset +'_split_'+str(self.split)+'_tsn_rgb_reference_bn_inception.caffemodel' self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.image_sub = rospy.Subscriber(self.videotopic, Image,self.callback,queue_size=1) #print('Dum') return []
class tsn_classifier: def __init__(self): global mypath # services provided self.reconfig_srv_ = rospy.Service('reconf_split',split, self.reconfig_srv) self.start_vidscores = rospy.Service('start_vidscores', Empty, self.start_vidscores) self.stop_vidscores = rospy.Service('stop_vidscores', Empty, self.stop_vidscores) # topics published # self.image_pub = rospy.Publisher("class_overlay_image_raw",Image, queue_size=1) # self.label_fw_pub = rospy.Publisher("action_fw", String, queue_size=1) # self.label_pub = rospy.Publisher("action", String, queue_size=1) # self.ownlabel_pub = rospy.Publisher("action_own", String, queue_size=1) # parameters self.dataset = rospy.get_param('~dataset','hmdb51') self.device_id = rospy.get_param('~device_id',0) self.split = rospy.get_param('~split',1) self.step = rospy.get_param('~step',6) # this should actually be # step = (frame_cnt - stack_depth) / (args.num_frame_per_video-1) # it will change depending on the action length, a value I don't have if I am classifying real time, but that I could get if I am doing it by service calls! self.stack_depth = rospy.get_param('~stack_depth',5) # stack_depth is 1 for rgb and 5 for flows. I am letting it be 5 to test creating an array of cv_images self.classwindow = rospy.get_param('~classification_frame_window',50) #whatswrong = (rospy.resolve_name('~action_list')) #rospy.spin() self.actionlist = rosparam.get_param(rospy.resolve_name('~action_list')) #"['brush_hair','cartwheel','catch','chew','clap','climb','climb_stairs','dive','draw_sword','dribble','drink','eat','fall_floor','fencing','flic_flac','golf','handstand','hit','hug','jump','kick','kick_ball','kiss','laugh','pick','pour','pullup','punch','push','pushup','ride_bike','ride_horse','run','shake_hands','shoot_ball','shoot_bow','shoot_gun','sit','situp','smile','smoke','somersault','stand','swing_baseball','sword','sword_exercise','talk','throw','turn','walk','wave']") #if type(self.actionlist) is str: # self.actionlist = eval(self.actionlist) self.actionlist.sort() self.chooselist = rosparam.get_param(rospy.resolve_name('~choose_list')) ## I must be doing something wrong here for this name not to be resolved. maybe it is because each node here should probably have its own init_node and it doesn't #if type(self.chooselist) is str: # self.chooselist = eval(self.chooselist) self.chooselist.sort() ###probably should use the nice rosparam thingy here to avoid these problems... self.framesize_width = rospy.get_param('~framesize_width',340) self.framesize_height = rospy.get_param('~framesize_height',256) # topics subscribed self.image_sub = rospy.Subscriber('video_topic', Image,self.callback,queue_size=1) # internals self.bridge = CvBridge() from pyActionRecog.utils.video_funcs import default_aggregation_func if self.chooselist: keepi = [] rospy.logwarn('defined own subset of actions! classification will be reduced to smaller set of choices, namely:'+str(self.chooselist)) #print(range(0,len(self.actionlist))) for i in range(0,len(self.actionlist)): for j in range(0, len(self.chooselist)): #print(self.actionlist[i]) #print( self.chooselist[j]) if self.actionlist[i] == self.chooselist[j]: keepi.append(i) tobedeleted = set(range(0,len(self.actionlist)))-set(keepi) #print(tobedeleted) self.defprox = lambda x: np.delete(default_aggregation_func(x),list(tobedeleted)) self.actionlist = self.chooselist else: rospy.logwarn('No choose_list defined. Will classify within the whole set. ') self.defprox = default_aggregation_func self.frame_scores = [] self.prototxt = mypath+'/models/'+ self.dataset +'/tsn_bn_inception_rgb_deploy.prototxt' self.caffemodel = mypath+'/models/'+ self.dataset +'_split_'+str(self.split)+'_tsn_rgb_reference_bn_inception.caffemodel' self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.ownvidscores = [] # when I instantiate the classifier, the startedownvid is working already. this influences how vsmf_srv will behave, so it needs to be like this, I think. self.startedownvid = True self.lock = threading.Lock() #publishers self.label_fw_pub = FunnyPublisher("action_fw", self.actionlist, self.defprox) self.label_pub = FunnyPublisher("action", self.actionlist, self.defprox) self.ownlabel_pub = FunnyPublisher("action_own", self.actionlist, self.defprox) rospy.set_param('~alive',0.5) rospy.loginfo("waiting for callback from " +rospy.resolve_name('video_topic') +" to do anything") def start_vidscores(self,req): # I will need to use locks here, I think... with self.lock: self.startedownvid = True rospy.logwarn("Started classifying own vid!") return [] def stop_vidscores(self,req): # I will need to use locks here, I think... with self.lock: self.startedownvid = False if self.ownvidscores: self.ownlabel_pub.pub(self.ownvidscores) else: rospy.logerr('ownvidscores is empty!!!!!!!!!!!!!!! are we locking for too long?') self.ownvidscores = [] rospy.logdebug("published the label for the own video version!") rospy.logwarn("stopped classifying own vid") return [] def callback(self,data): rospy.logdebug("reached callback. that means I can read the Subscriber!") rospy.set_param('~alive',1) try: cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: print(e) scores = self.net.predict_single_frame([cv_image,], 'fc-action', frame_size=(self.framesize_width, self.framesize_height)) #print((scores)) #this publishes the instant time version, aka, per frame self.label_pub.pub([scores]) rospy.logdebug("published the label for instant time version!") #this part publishes the frame_window version self.frame_scores.append(scores) if len(self.frame_scores)>self.classwindow: self.frame_scores.pop(0) self.label_fw_pub.pub(self.frame_scores) rospy.logdebug("published the label for the frame window version!") self.lock.acquire() if self.startedownvid: self.ownvidscores.append(scores) else: rospy.logdebug_throttle(20,"waiting for start_vidscores call to start classifying ownvid") self.lock.release() # try: # self.image_pub.publish(self.bridge.cv2_to_imgmsg(cv_image, "bgr8")) # rospy.logdebug("published image") # except CvBridgeError as e: # print(e) def reconfig_srv(self, req): # why not use standard ros reconfigure stds? self.image_sub.unregister() self.split = req.Split rospy.loginfo("reading split:"+str(req.Split)) #print(req.Split) self.caffemodel = mypath+'/models/'+ self.dataset +'_split_'+str(self.split)+'_tsn_rgb_reference_bn_inception.caffemodel' self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.image_sub = rospy.Subscriber(self.videotopic, Image,self.callback,queue_size=1) #print('Dum') return []
def __init__(self): global mypath # services provided self.reconfig_srv_ = rospy.Service('reconf_split',split, self.reconfig_srv) self.start_vidscores = rospy.Service('start_vidscores', Empty, self.start_vidscores) self.stop_vidscores = rospy.Service('stop_vidscores', Empty, self.stop_vidscores) # topics published # self.image_pub = rospy.Publisher("class_overlay_image_raw",Image, queue_size=1) # self.label_fw_pub = rospy.Publisher("action_fw", String, queue_size=1) # self.label_pub = rospy.Publisher("action", String, queue_size=1) # self.ownlabel_pub = rospy.Publisher("action_own", String, queue_size=1) # parameters self.dataset = rospy.get_param('~dataset','hmdb51') self.device_id = rospy.get_param('~device_id',0) self.split = rospy.get_param('~split',1) self.step = rospy.get_param('~step',6) # this should actually be # step = (frame_cnt - stack_depth) / (args.num_frame_per_video-1) # it will change depending on the action length, a value I don't have if I am classifying real time, but that I could get if I am doing it by service calls! self.stack_depth = rospy.get_param('~stack_depth',5) # stack_depth is 1 for rgb and 5 for flows. I am letting it be 5 to test creating an array of cv_images self.classwindow = rospy.get_param('~classification_frame_window',50) #whatswrong = (rospy.resolve_name('~action_list')) #rospy.spin() self.actionlist = rosparam.get_param(rospy.resolve_name('~action_list')) #"['brush_hair','cartwheel','catch','chew','clap','climb','climb_stairs','dive','draw_sword','dribble','drink','eat','fall_floor','fencing','flic_flac','golf','handstand','hit','hug','jump','kick','kick_ball','kiss','laugh','pick','pour','pullup','punch','push','pushup','ride_bike','ride_horse','run','shake_hands','shoot_ball','shoot_bow','shoot_gun','sit','situp','smile','smoke','somersault','stand','swing_baseball','sword','sword_exercise','talk','throw','turn','walk','wave']") #if type(self.actionlist) is str: # self.actionlist = eval(self.actionlist) self.actionlist.sort() self.chooselist = rosparam.get_param(rospy.resolve_name('~choose_list')) ## I must be doing something wrong here for this name not to be resolved. maybe it is because each node here should probably have its own init_node and it doesn't #if type(self.chooselist) is str: # self.chooselist = eval(self.chooselist) self.chooselist.sort() ###probably should use the nice rosparam thingy here to avoid these problems... self.framesize_width = rospy.get_param('~framesize_width',340) self.framesize_height = rospy.get_param('~framesize_height',256) # topics subscribed self.image_sub = rospy.Subscriber('video_topic', Image,self.callback,queue_size=1) # internals self.bridge = CvBridge() from pyActionRecog.utils.video_funcs import default_aggregation_func if self.chooselist: keepi = [] rospy.logwarn('defined own subset of actions! classification will be reduced to smaller set of choices, namely:'+str(self.chooselist)) #print(range(0,len(self.actionlist))) for i in range(0,len(self.actionlist)): for j in range(0, len(self.chooselist)): #print(self.actionlist[i]) #print( self.chooselist[j]) if self.actionlist[i] == self.chooselist[j]: keepi.append(i) tobedeleted = set(range(0,len(self.actionlist)))-set(keepi) #print(tobedeleted) self.defprox = lambda x: np.delete(default_aggregation_func(x),list(tobedeleted)) self.actionlist = self.chooselist else: rospy.logwarn('No choose_list defined. Will classify within the whole set. ') self.defprox = default_aggregation_func self.frame_scores = [] self.prototxt = mypath+'/models/'+ self.dataset +'/tsn_bn_inception_rgb_deploy.prototxt' self.caffemodel = mypath+'/models/'+ self.dataset +'_split_'+str(self.split)+'_tsn_rgb_reference_bn_inception.caffemodel' self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.ownvidscores = [] # when I instantiate the classifier, the startedownvid is working already. this influences how vsmf_srv will behave, so it needs to be like this, I think. self.startedownvid = True self.lock = threading.Lock() #publishers self.label_fw_pub = FunnyPublisher("action_fw", self.actionlist, self.defprox) self.label_pub = FunnyPublisher("action", self.actionlist, self.defprox) self.ownlabel_pub = FunnyPublisher("action_own", self.actionlist, self.defprox) rospy.set_param('~alive',0.5) rospy.loginfo("waiting for callback from " +rospy.resolve_name('video_topic') +" to do anything")
def build_net(net_proto, net_weights): global net net = CaffeNet(net_proto, net_weights, 0)
class tsn_classifier: def __init__(self): global mypath # services provided self.reconfig_srv_ = rospy.Service('reconf_split', split, self.reconfig_srv) self.start_vidscores = rospy.Service('start_vidscores', Empty, self.start_vidscores) self.stop_vidscores = rospy.Service('stop_vidscores', Empty, self.stop_vidscores) # topics published self.scores_pub = rospy.Publisher("scores", Float32MultiArray, queue_size=1) self.score_fw_pub = rospy.Publisher("scores_fw", caffe_tsn_ros.msg.ScoreArray, queue_size=1) # self.label_pub = rospy.Publisher("action", String, queue_size=1) self.ownlabel_pub = rospy.Publisher("scores_own", caffe_tsn_ros.msg.ScoreArray, queue_size=1) # parameters self.dataset = rospy.get_param('~dataset', 'hmdb51') self.device_id = rospy.get_param('~device_id', 0) self.split = rospy.get_param('~split', 1) self.rgbOrFlow = rospy.get_param('~classifier_type') self.clear_fw_after_new_vid = rospy.get_param( '~clear_fw_after_new_vid', True) self.step = rospy.get_param('~step', 6) # this should actually be # step = (frame_cnt - stack_depth) / (args.num_frame_per_video-1) # it will change depending on the action length, a value I don't have if I am classifying real time, but that I could get if I am doing it by service calls! # stack_depth is 1 for rgb and 5 for flows. I am letting it be 5 to test creating an array of cv_images self.stack_depth = rospy.get_param('~stack_depth', 5) self.stack_count = 0 self.cv_image_stack = [] self.classwindow = rospy.get_param('~classification_frame_window', 'undefined') assert not self.classwindow == 'undefined' ###probably should use the nice rosparam thingy here to avoid these problems... self.framesize_width = rospy.get_param('~framesize_width', 340) self.framesize_height = rospy.get_param('~framesize_height', 256) # topics subscribed self.image_sub = rospy.Subscriber('video_topic', Image, self.callback, queue_size=1) # internals self.bridge = CvBridge() self.prototxt = mypath + '/models/' + self.dataset + '/tsn_bn_inception_' + self.rgbOrFlow + '_deploy.prototxt' self.caffemodel = mypath + '/models/' + self.dataset + '_split_' + str( self.split ) + '_tsn_' + self.rgbOrFlow + '_reference_bn_inception.caffemodel' rospy.loginfo("loading prototxt {}".format(self.prototxt)) rospy.loginfo("loading caffemodel {}".format(self.caffemodel)) self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.frame_scores = caffe_tsn_ros.msg.ScoreArray() self.ownvidscores = caffe_tsn_ros.msg.ScoreArray() # when I instantiate the classifier, the startedownvid is working already. this influences how vsmf_srv will behave, so it needs to be like this, I think. self.startedownvid = True self.lock = threading.Lock() self.published_fw = False rospy.set_param('~alive', 0.5) rospy.loginfo("waiting for callback from " + rospy.resolve_name('video_topic') + " to do anything") def start_vidscores(self, req): # I will need to use locks here, I think... with self.lock: self.startedownvid = True self.published_fw = False rospy.loginfo("Started classifying own vid!") return [] def stop_vidscores(self, req): # I will need to use locks here, I think... with self.lock: self.startedownvid = False if self.ownvidscores: #### I am going to publish now a set of matrices, right? self.ownlabel_pub.publish(self.ownvidscores) rospy.logdebug("length of ownvidscores:%d" % (len(self.ownvidscores.data))) #pass else: rospy.logerr( 'ownvidscores is empty!!!!!!!!!!!!!!! are we locking for too long?' ) self.ownvidscores = caffe_tsn_ros.msg.ScoreArray() if not self.published_fw: rospy.logwarn("did not publish a single fw for this video") rospy.logwarn("length of fw:%d" % (len(self.frame_scores.data))) if self.clear_fw_after_new_vid: self.frame_scores = caffe_tsn_ros.msg.ScoreArray() rospy.logdebug("published the label for the own video version!") rospy.loginfo("stopped classifying own vid") return [] def callback(self, data): rospy.logdebug( "reached callback. that means I can read the Subscriber!") rospy.set_param('~alive', 1) try: cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: #print(e) rospy.logerr(e) #since I am not using stacks for rgb images, I can prevent from making the rgb version any slower by using an if statement here if self.rgbOrFlow == 'flow': ## and I want the combined flow version here, don't I? so I need to strip the frame apart into components. I think it is better than #self.cv_image_stack.append(cv_image) #this would be incorrect, i need to get each matrix and put it together in a stack. first x then y # from ros_flow_bg, I can see that x is the 0 component and y the 1 component. I hope bgr8 stays bgr8 and we don't flip things around here! self.cv_image_stack.append(cv_image[:, :, 0]) self.cv_image_stack.append(cv_image[:, :, 1]) # self.cv_image_stack.extend([cv_image[:,:,0], cv_image[:,:,1]]) if len( self.cv_image_stack ) > 2 * self.stack_depth: #keeps at most 2*self.stack_depth images in cv_image_stack, the 2 comes from the fact that we are using flow_x and flow_y self.cv_image_stack.pop(0) self.cv_image_stack.pop(0) if self.stack_count % self.step == 0: rospy.logdebug("reached execution part of callback!") self.stack_count = 0 ## we don't keep a large number here. scores = None ### i can maybe use a lambda to abstract this. is it faster than an if though? if self.rgbOrFlow == 'rgb': scores = self.net.predict_single_frame( [ cv_image, ], 'global_pool', frame_size=(self.framesize_width, self.framesize_height)) elif self.rgbOrFlow == 'flow' and len(self.cv_image_stack) == 10: scores = self.net.predict_single_flow_stack( self.cv_image_stack, 'global_pool', frame_size=(self.framesize_width, self.framesize_height)) #print(type(scores)) #print(scores.dtype) #scoremsg = caffe_tsn_ros.msg.Scores() #scoremsg.test = 'hello' #scoremsg.scores = scores #print(scoremsg) #scores = np.squeeze(scores) #scores = np.array([[[1.,2.],[3.,4.],[5.,6.]],[[11.,12.],[13.,14.],[15.,16.]]],dtype='float32') #self.scores_pub.publish(self.bridge.cv2_to_imgmsg(scores, '32FC1')) #self.scores_pub.publish(scoremsg) #print(self.scores_pub.get_num_connections()) #print((scores)) #print(np.shape(scores)) #print(scores.shape) if isinstance(scores, np.ndarray): #this publishes the instant time version, aka, per frame #self.label_pub.pub([scores]) ## Not sure if this MultiArrayDimension thing is working. In any ## case, it is already a LOT of data per frame; publishing the chunk ## at the end of the video, all at the same time would probably ## cause a lot of unnecessary waiting # TO CONSIDER: this is fast, so I think it doesn't matter, but I # believe the layout could be pre-set, since it doesn't change on a # frame by frame basis. myheader = Header() myheader.stamp = rospy.Time.now() scoresmsg = Float32MultiArray() scoresmsg.layout.dim = [] dims = np.array(scores.shape) scoresize = dims.prod() / float(scores.nbytes) for i in range(0, len(dims)): #print(i) scoresmsg.layout.dim.append(MultiArrayDimension()) scoresmsg.layout.dim[i].size = dims[i] scoresmsg.layout.dim[i].stride = dims[i:].prod( ) / scoresize scoresmsg.layout.dim[i].label = 'dim_%d' % i #print(scoresmsg.layout.dim[i].size) #print(scoresmsg.layout.dim[i].stride) scoresmsg.data = np.frombuffer(scores.tobytes(), 'float32') self.scores_pub.publish(scoresmsg) #rospy.logdebug("published the label for instant time version!") #this part publishes the frame_window version self.frame_scores.data.append(scoresmsg) #rospy.logwarn("FISHY: length of fw:%d"%(len(self.frame_scores.data))) #rospy.logwarn("FISHY: self.classwindow:%d"%self.classwindow) if len(self.frame_scores.data) > self.classwindow: #rospy.logwarn("FISHY: got into the loop") self.frame_scores.header = myheader self.frame_scores.data.pop(0) self.score_fw_pub.publish(self.frame_scores) #rospy.logwarn("FISHY: passed the publisher, so it must have worked?") with self.lock: self.published_fw = True ## it was not working without the lock rospy.logdebug( "published the label for the frame window version!") with self.lock: if self.startedownvid: self.ownvidscores.data.append(scoresmsg) self.ownvidscores.header = myheader #pass else: rospy.logdebug_throttle( 20, "waiting for start_vidscores call to start classifying ownvid" ) self.stack_count = self.stack_count + 1 # try: # self.image_pub.publish(self.bridge.cv2_to_imgmsg(cv_image, "bgr8")) # rospy.logdebug("published image") # except CvBridgeError as e: # print(e) def reconfig_srv(self, req): # why not use standard ros reconfigure stds? self.image_sub.unregister() self.split = req.Split rospy.loginfo("reading split:" + str(req.Split)) #print(req.Split) self.caffemodel = mypath + '/models/' + self.dataset + '_split_' + str( self.split ) + '_tsn_' + self.rgbOrFlow + '_reference_bn_inception.caffemodel' self.net = CaffeNet(self.prototxt, self.caffemodel, self.device_id) self.image_sub = rospy.Subscriber('video_topic', Image, self.callback, queue_size=1) #print('Dum') return []