def get_text_ranks():
segmenter = Segmenter()
stopwords = get_stopwords()
print("Start TextRank over the selected quatrains ...")
corpus = get_corpus()
adjlist = dict()
for idx, poem in enumerate(corpus):
if 0 == (idx + 1) % 10000:
print("[TextRank] Scanning %d/%d poems ..." %
(idx + 1, len(corpus)))
for sentence in poem['sentence']:
segs = list(
filter(lambda word: word not in stopwords,
segmenter.segment(sentence)))
for seg in segs:
if seg not in adjlist:
adjlist[seg] = dict()
for i, seg in enumerate(segs):
for _, other in enumerate(segs[i + 1:]):
if seg != other:
adjlist[seg][other] = adjlist[seg][other] + 1 \
if other in adjlist[seg] else 1.0
adjlist[other][seg] = adjlist[other][seg] + 1 \
if seg in adjlist[other] else 1.0
for word in adjlist:
w_sum = sum(weight for other, weight in adjlist[word].items())
for other in adjlist[word]:
adjlist[word][other] /= w_sum
print("[TextRank] Weighted graph has been built.")
_text_rank(adjlist)
def prepare_training_data():
word_ranks = get_word_ranks()
corpus = get_corpus()
segmenter = Segmenter()
with codecs.open('./data/training.txt', 'w', 'utf-8') as fout:
for poem in corpus:
poem['keyword'] = []
stop=False
for sentence in poem['sentence']:
segs = list(filter(lambda seg: seg in word_ranks, segmenter.segment(sentence)))
if len(segs) == 0:
stop = True
break
if len(poem['sentence'])!=4 or stop:
continue
for sentence in poem['sentence']:
segs = list(filter(lambda seg: seg in word_ranks, segmenter.segment(sentence)))
if len(segs) == 0:
print('aaa', sentence)
keyword = reduce(lambda x,y: x if word_ranks[x]>word_ranks[y] else y, segs)
poem['keyword'].append(keyword)
if(len(keyword)>=2):
print(sentence, keyword)
fout.write(sentence + '\t' + keyword + '\n')
def applySegmentation(metafilename,img,device='/gpu:0',conf=None):
'''
Loads the graph from the meta file `metafilename' and predicts segmentations for the image stack `img'. The graph is
expected to have a collection "endpoints" storing the x,y_,y,ypred list of tensors where x is the input and ypred the
predicted segmentation. The first 2 dimensions of `img' must be the XY dimensions, other dimensions are flattened out.
The values of `img' are also expected to be normalized. Returns a stack of binary masks with the same shape as `img'.
'''
seg=Segmenter(metafilename,device,conf)
origshape=tuple(img.shape)
tf.logging.info('Input dimensions: %r'%(origshape,))
shape=origshape+(1,1,1) # add extra dimensions to the shape to make a 4D shape
shape=shape[:5] # clip extra dimensions off so that this is a 5D shape description
width,height,slices,timesteps,depth=shape
img=img.astype(np.dtype('<f8')).reshape(shape) # convert input into a 5D image of shape XYZTC
img=rescaleArray(img)
for s in range(slices):
tf.logging.info('Segmenting slice %s'%s)
for t in range(timesteps):
st=img[...,s,t,:]
if st.max()>st.min(): # segment if the image is not blank
pred=seg.apply(st)
img[...,s,t,:]=0
img[...,s,t,0]=pred
return img.reshape(origshape)
def __init__(self, tagger_pickle = None):
if tagger_pickle == None:
path = os.path.abspath(__file__)\
.replace(os.path.basename(__file__), '')
tagger_pickle = \
'%smodels/sinica_treebank_brill_aubt.pickle' % (path)
self.tagger = pickle.load(open(tagger_pickle))
self.segmenter = Segmenter()
def main():
img_size = 505
gpu_id, net_path, model_path, img_paths = process_arguments(sys.argv)
palette = pascal_palette_invert()
net = Segmenter(net_path, model_path, gpu_id)
for img_path in img_paths:
img, cur_h, cur_w = preprocess_image(img_path, img_size)
segm_result = net.predict([img])
segm_post = postprocess_segmentation(segm_result, cur_h, cur_w, palette)
concatenate = True
segm_name = os.path.basename(img_path).split('.')[0]+'-label.png'
save_result(segm_post, segm_name, concatenate, img_path)
def __init__(self, app, config=None):
self.app = app
ngrams_file = config.get('segmenter', 'ngrams_file')
self.app.log.debug(f'Trying to load ngrams file {ngrams_file}')
try:
with open(ngrams_file, 'r') as nf:
ngrams = json.load(nf)
self.app.log.debug(f'Loaded ngrams file {ngrams_file}.')
except FileNotFoundError as err:
self.app.log.info(
f'Ngrams file {ngrams_file} not found. Using default configuration.'
)
self._ws = Segmenter(ngrams)
def main():
img_size = 505
gpu_id, net_path, model_path, img_paths = process_arguments(sys.argv)
palette = pascal_palette_invert()
net = Segmenter(net_path, model_path, gpu_id)
for img_path in img_paths:
img, cur_h, cur_w = preprocess_image(img_path, img_size)
segm_result = net.predict([img])
segm_post = postprocess_segmentation(segm_result, cur_h, cur_w,
palette)
concatenate = True
segm_name = os.path.basename(img_path).split('.')[0] + '-label.png'
save_result(segm_post, segm_name, concatenate, img_path)
def run(self):
base_preprocessor = Preprocessor(self.basefile)
base_preprocessed_image_stack = base_preprocessor.preprocess_basefile()
arc_preprocessor = Preprocessor(self.arcfile)
arc_preprocessed_image_stack = arc_preprocessor.preprocess_arcfile()
x, y, z = self.cell_aggragation_shape
segmenter = Segmenter(base_preprocessed_image_stack,
self.create_np_ellipsoid(x, y, z),
"WS",
generate_debugging=self.enable_debugging)
segmented_image_stack = segmenter.run_segmentation()
analysis = Analysis(segmented_image_stack,
arc_preprocessed_image_stack)
analysis.generate_report()
colorized_image_stack = analysis.colorize_overlapping_cells()
self.save_image_stack(colorized_image_stack)
def segment(self, image, slow=False):
def predictor(X):
X = np.expand_dims(X, axis=0)
y = self.predict(X)
y = np.squeeze(y, axis=0)
return y
if slow:
input_height = self.S.image_height
input_width = self.S.image_width
else:
_, input_height, input_width = image.shape
segmenter = Segmenter(predictor,
self.S.image_depth,
input_height,
input_width,
image)
return segmenter.predict()
def __init__(self, model_file):
self.max_success_count = 0
self.success_count = 0
self.dropbox_dict = {}
self.dict_list = []
self.object_list = None
self.color_list = []
self.yaml_saved = False
self.collision_map_complete = False
self.goal_positions = [0] #[0, -1.0, 1.0] uncomment to rotate robot
self.table_cloud = None
self.collision_map_base_list = []
self.detected_objects = None
#load SVM object classifier
self.model = pickle.load(open(model_file, 'rb'))
#initialize object segmenter
self.segmenter = Segmenter(self.model)
#initialize point cloud publishers
self.objects_pub = rospy.Publisher("/pcl_objects",
PointCloud2,
queue_size=1)
#self.table_pub = rospy.Publisher("/pcl_table", PointCloud2, queue_size=1)
#self.colorized_cluster_pub = rospy.Publisher("/colorized_clusters", PointCloud2, queue_size=1)
self.object_markers_pub = rospy.Publisher("/object_markers",
Marker,
queue_size=1)
self.detected_objects_pub = rospy.Publisher("/detected_objects",
DetectedObjectsArray,
queue_size=1)
self.denoised_pub = rospy.Publisher("/denoised_cloud",
PointCloud2,
queue_size=1)
#self.reduced_cloud_pub = rospy.Publisher("/decimated_cloud", PointCloud2, queue_size = 1)
self.collision_cloud_pub = rospy.Publisher("/pr2/3d_map/points",
PointCloud2,
queue_size=1)
print('PR2 object initialized.')
def fetch_nonlabeled_data():
proj_name = "article150801160830"
str_old_time = "2015-08-01 00:00:00"
str_new_time = "2016-12-31 00:00:00"
# Crawler(proj_name=proj_name).rebuild_table()
# Crawler163(proj_name=proj_name).crawl(str_old_time, str_new_time)
# Crawler36Kr(proj_name=proj_name).crawl(str_old_time, str_new_time)
# CrawlerGeekPark(proj_name=proj_name).crawl(str_old_time, str_new_time)
# CrawlerLeiphone(proj_name=proj_name).crawl(str_old_time, str_new_time)
# CrawlerKanchai(proj_name=proj_name).crawl(str_old_time, str_new_time)
# CrawlerHuxiu(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawlerIheima(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawlerKanchai(proj_name=proj_name).crawl(str_old_time,
str_new_time)
LabeledCrawlerLeiphone(proj_name=proj_name).crawl(str_old_time,
str_new_time)
LabeledCrawlerLieyun(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawlerSootoo(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawlerYiou(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawler7tin(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
LabeledCrawlerAilab(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
LabeledCrawlerBaidu(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
LabeledCrawlerSinaVR(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
LabeledCrawlerVarkr(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
seg = Segmenter(proj_name=proj_name)
seg.seg(skip_exist=True)
seg.join_segfile()
def fetch_labeled_data():
str_old_time = "2015-08-01 00:00:00"
str_new_time = "2016-11-31 00:00:00"
proj_name = "article_cat"
LabeledCrawler(proj_name=proj_name).rebuild_table()
LabeledCrawlerIheima(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawlerKanchai(proj_name=proj_name).crawl(str_old_time,
str_new_time)
LabeledCrawlerLeiphone(proj_name=proj_name).crawl(str_old_time,
str_new_time)
LabeledCrawlerLieyun(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawlerSootoo(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawlerYiou(proj_name=proj_name).crawl(str_old_time, str_new_time)
LabeledCrawler7tin(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
LabeledCrawlerAilab(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
LabeledCrawlerBaidu(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
LabeledCrawlerSinaVR(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
LabeledCrawlerVarkr(proj_name=proj_name).crawl("2000-08-01 00:00:00",
str_new_time)
seg = Segmenter(proj_name=proj_name)
seg.seg(skip_exist=True)
seg.join_segfile()
class Renamer():
lower_list = [
"a", "an", "the", "and", "but", "or", "for", "nor", "with", "to", "on",
"as", "at", "by", "in", "of", "mid", "off", "per", "qua", "re", "up",
"via", "o'", "'n'", "n'"
]
def __init__(self, app, config=None):
self.app = app
ngrams_file = config.get('segmenter', 'ngrams_file')
self.app.log.debug(f'Trying to load ngrams file {ngrams_file}')
try:
with open(ngrams_file, 'r') as nf:
ngrams = json.load(nf)
self.app.log.debug(f'Loaded ngrams file {ngrams_file}.')
except FileNotFoundError as err:
self.app.log.info(
f'Ngrams file {ngrams_file} not found. Using default configuration.'
)
self._ws = Segmenter(ngrams)
def suggest_correction(self, filepath):
filename = os.path.basename(filepath)
filename, ext = os.path.splitext(filename)
# Look for ending '2e', '3e' etc giving edition number
edition_match = re.match('(.*)(\d)e$', filename)
filename, edition = edition_match.groups() if edition_match else (
filename, '')
result_segments = []
# Process each segment individually
for token in filename.split('_'):
words = self._ws.segment(token)
# To title case
words = [words[0][:1].upper() + words[0][1:]] + [
(word[:1].upper() if word not in self.lower_list else word[:1])
+ word[1:] for word in words[1:]
]
result_segments.append(' '.join(words))
# Join suggestions for segments
result = " - ".join(result_segments)
# Add edition information
if edition:
result = result + ' ({} edition)'.format(
num2words(edition, to='ordinal_num'))
return result + ext
class POSTagger():
def __init__(self, tagger_pickle = None):
if tagger_pickle == None:
path = os.path.abspath(__file__)\
.replace(os.path.basename(__file__), '')
tagger_pickle = \
'%smodels/sinica_treebank_brill_aubt.pickle' % (path)
self.tagger = pickle.load(open(tagger_pickle))
self.segmenter = Segmenter()
def tag(self, text):
tokens = self.segmenter.segmentation(text)
return self.tagger.tag(tokens)
def test_net(net_path, model_path, images, labels, lut, gpu_id):
net = Segmenter(net_path, model_path, gpu_id)
mean_vec = np.array([103.939, 116.779, 123.68], dtype=np.float32)
reshaped_mean_vec = mean_vec.reshape(1, 1, 3);
pa_list = []
ma_list = []
m_IU_list = []
fw_IU_list = []
pb = ProgressBar(len(images))
for img_path, label_path in zip(images, labels):
im, cur_h, cur_w = preprocess_image(img_path, reshaped_mean_vec)
label = imread(label_path)
label = lut[label]
segmentation = net.predict([im])
pred = segmentation[0:cur_h, 0:cur_w]
pa = pixel_accuracy(pred, label)
ma = mean_accuracy(pred, label)
m_IU = mean_IU(pred, label)
fw_IU = frequency_weighted_IU(pred, label)
pa_list.append(pa)
ma_list.append(ma)
m_IU_list.append(m_IU)
fw_IU_list.append(fw_IU)
pb.print_progress()
print("pixel_accuracy: " + str(np.mean(pa_list)))
print("mean_accuracy: " + str(np.mean(ma_list)))
print("mean_IU: " + str(np.mean(m_IU_list)))
print("frequency_weighted: " + str(np.mean(fw_IU_list)))
def load_segmenter(self):
training_file_name = os.path.splitext(
self.settings['training_file'])[0]
outpath = training_file_name + '.segmenter.pickle'
segmenter = None
if os.path.exists(outpath):
with open(outpath, 'rb') as handle:
segmenter = pickle.load(handle)
else:
if outpath is not None:
segmenter = Segmenter(self.settings['training_file'])
with open(outpath, 'wb') as f:
pickle.dump(segmenter, f)
print("Segmenter model written out to {}".format(outpath))
return segmenter
def main(input_path, output_path):
imgs_path = sorted(glob(f'{input_path}/*'))
for img_path in imgs_path:
img_name = img_path.split('/')[-1].split('.')[0]
out_file = open(f'{output_path}/{img_name}.txt', "w")
print(f"Processing new image {img_name}...")
img = io.imread(img_path)
img = gray_img(img)
horizontal = IsHorizontal(img)
if horizontal == False:
theta = deskew(img)
img = rotation(img, theta)
img = get_gray(img)
img = get_thresholded(img, threshold_otsu(img))
img = get_closer(img)
horizontal = IsHorizontal(img)
original = img.copy()
gray = get_gray(img)
bin_img = get_thresholded(gray, threshold_otsu(gray))
segmenter = Segmenter(bin_img)
imgs_with_staff = segmenter.regions_with_staff
most_common = segmenter.most_common
# imgs_without_staff = segmenter.regions_without_staff
imgs_spacing = []
imgs_rows = []
coord_imgs = []
for i, img in enumerate(imgs_with_staff):
spacing, rows, no_staff_img = coordinator(img, horizontal)
imgs_rows.append(rows)
imgs_spacing.append(spacing)
coord_imgs.append(no_staff_img)
print("Recognize...")
recognize(out_file, most_common, coord_imgs,
imgs_with_staff, imgs_spacing, imgs_rows)
out_file.close()
print("Done...")
def main():
# default fpaths
ngrams1 = "1grams.txt"
ngrams2 = "2grams.txt"
# parse arguments
args = sys.argv[1:]
for arg in args:
if arg == "-simple":
ngrams2 = None
# check existence of 1grams
if not os.path.isfile("1grams.txt"):
print("Error: could not find 1grams.txt.")
print("Exiting....")
sys.exit(1)
# check existence of 2grams and 3grams
if ngrams2 and not os.path.isfile(ngrams2):
print("Could not find some files.")
print("Segmenter will run without use of 2grams and 3grams.")
ngrams2 = None
# make segmenter
global __seg
__seg = Segmenter(ngrams1, fpath_2grams=ngrams2)
# show intro msg
clear()
global __INTRO
print(__INTRO)
print("hello, world!")
# read, eval
while (True):
user_input = str(input("> "))
eval(user_input)
"cache_dir", "../../models"))
model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h",
cache_dir=os.getenv(
"cache_dir", "../../models"))
audio_ds = [
os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data',
'sample.mp3'),
os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data',
'long_sample.mp3')
]
# create speech segmenter
seg = Segmenter(model_path=os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'speech_segmenter_models'),
vad_engine='smn',
detect_gender=True,
ffmpeg='ffmpeg',
batch_size=32)
# it holds audio segmentations
segmentations = []
for audio in audio_ds:
# [('noEnergy', 0.0, 0.8), ('male', 0.8, 9.84), ('music', 9.84, 10.96), ('male', 10.96, 14.98)]
# segmentation = seg(audio, start_sec=0, stop_sec=30)
s = seg(audio)
res = {}
res['segmentation'] = s
res['audio'] = audio
segmentations.append(res)
def thread_func(args, detector, predictor, img_queue, result_queue):
"""Get face from image queue. This function is used for multiprocessing"""
# Introduce mark_detector to detect landmarks.
gaze_model = args["gaze_net"]
eye_size = args["eye_size"]
face_size = args["face_size"]
inputs = args["inputs"]
outputs = args["outputs"]
print("[INFO] loading gaze predictor...")
gaze_detector = GazeEstimator(gaze_model=gaze_model,
eye_image_size=eye_size,
face_image_size=face_size,
inputs=inputs,
outputs=outputs)
# init variables
detectorWidth = 400
faceBoxScale = 0.15
while True:
# get the image
try:
frame = img_queue.get(timeout=1)
except Q.Empty:
print("Image Q empty, thread exiting!")
return
# update factors
originalWidth = frame.shape[1]
factor = originalWidth / detectorWidth
# resize for face detection
image = imutils.resize(frame, width=detectorWidth)
# convert to grayscale for face detection
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# actually run face detection
faceboxes, scores, idx = detector.run(image, 0)
if faceboxes is not None and len(faceboxes) > 0:
facebox = faceboxes[0]
confidence = scores[0]
# get 5 landmarks
marks = predictor(gray, facebox)
# convert marks to np array
marks = face_utils.shape_to_np(marks)
leftEyeMarks = []
rightEyeMarks = []
# pull out left and right eye marks
for (i, (x, y)) in enumerate(marks):
[x, y] = [int(x * factor), int(y * factor)]
if i == 0 or i == 1:
leftEyeMarks.append([x, y])
if i == 2 or i == 3:
rightEyeMarks.append([x, y])
# convert the facebox from dlib format to regular BB and
# rescale it back to original image size
facebox = utils.dlib_to_box(facebox, factor, faceBoxScale)
# segment the image based on markers and facebox
seg = Segmenter(facebox, leftEyeMarks, rightEyeMarks,
frame.shape[1], frame.shape[0])
segments = seg.getSegmentJSON()
# detect gaze
gaze = gaze_detector.detect_gaze(frame, segments["leftEye"],
segments["rightEye"],
segments["face"],
segments["faceGrid"])
# pack result
result = [gaze, frame]
result_queue.put(result)
else:
result_queue.put(None)
def test_feature(self):
segmenter = Segmenter()
segmenter.use_labeling()
# Once the picture is segmented we beed to apply the zone feature on it
feature = DirectionZoneFeature()
feature.process_zones()
import glob
from PIL import Image
from segmenter import Segmenter
image_paths = glob.glob("input/*.png") # get all images in the input folder
segmenter = Segmenter() # create an instance of the classifier
predictions = segmenter.run_images(
image_paths) # run the classifier to get predictions
segmenter.save_predictions(predictions,
image_paths) # save the results to output folder
Image.fromarray(
predictions[0]).show() # show a single prediction (the first image)
#! python3
#! PY_PYTHON=3
import sys, os
from segmenter import Segmenter
if __name__ == "__main__":
seg = Segmenter(sys.argv)
class PR2(object):
def __init__(self, model_file):
self.max_success_count = 0
self.success_count = 0
self.dropbox_dict = {}
self.dict_list = []
self.object_list = None
self.color_list = []
self.yaml_saved = False
self.collision_map_complete = False
self.goal_positions = [0] #[0, -1.0, 1.0] uncomment to rotate robot
self.table_cloud = None
self.collision_map_base_list = []
self.detected_objects = None
#load SVM object classifier
self.model = pickle.load(open(model_file, 'rb'))
#initialize object segmenter
self.segmenter = Segmenter(self.model)
#initialize point cloud publishers
self.objects_pub = rospy.Publisher("/pcl_objects",
PointCloud2,
queue_size=1)
#self.table_pub = rospy.Publisher("/pcl_table", PointCloud2, queue_size=1)
#self.colorized_cluster_pub = rospy.Publisher("/colorized_clusters", PointCloud2, queue_size=1)
self.object_markers_pub = rospy.Publisher("/object_markers",
Marker,
queue_size=1)
self.detected_objects_pub = rospy.Publisher("/detected_objects",
DetectedObjectsArray,
queue_size=1)
self.denoised_pub = rospy.Publisher("/denoised_cloud",
PointCloud2,
queue_size=1)
#self.reduced_cloud_pub = rospy.Publisher("/decimated_cloud", PointCloud2, queue_size = 1)
self.collision_cloud_pub = rospy.Publisher("/pr2/3d_map/points",
PointCloud2,
queue_size=1)
print('PR2 object initialized.')
##YAML Utilities##
def send_to_yaml(self, yaml_filename, dict_list):
# Helper function to output to yaml file
data_dict = {"object_list": dict_list}
with open(yaml_filename, 'w') as outfile:
yaml.dump(data_dict, outfile, default_flow_style=False)
##PR2 mover##
def get_world_joint_state(self):
try:
msg = rospy.wait_for_message("joint_states",
JointState,
timeout=10)
index = msg.name.index('world_joint')
position = msg.position[index]
except rospy.ServiceException as e:
print('Failed to get world_joint position.')
position = 10**6
return position
def segment_scene(self, pcl_msg):
print('Initializing segmenter.')
seg = self.segmenter #to reduce verbosity below
# TODO: Convert ROS msg to PCL data
print('Converting ROS message to pcl format.')
cloud = ros_to_pcl(pcl_msg)
leaf_size = 0.005
# TODO: Voxel Grid Downsampling
print('Reducing voxel resolution.')
cloud = seg.voxel_grid_downsample(cloud, leaf_size=leaf_size)
#decimated_cloud = cloud
#Reduce outlier noise in object cloud
print('Rejecting outliers in raw cloud.')
cloud = seg.outlier_filter(cloud, 15, 0.01)
denoised_cloud = cloud
# TODO: PassThrough Filter
print('Applying passthrough filters.')
cloud = seg.axis_passthrough_filter(cloud, 'z',
(0.55, 2)) #filter below table
#passthroughz_cloud = cloud
cloud = seg.axis_passthrough_filter(
cloud, 'x', (.35, 10)) #filter out table front edge
#passthroughy_cloud = cloud
# TODO: RANSAC Plane Segmentation
# TODO: Extract inliers and outliers
print('Performing plane segmentation.')
table_cloud, objects_cloud = seg.ransac_plane_segmentation(
cloud, max_distance=leaf_size)
#Reduce outlier noise in object cloud
print('Rejecting outliers in objects cloud.')
objects_cloud = seg.outlier_filter(objects_cloud, 10, 0.01)
# TODO: Euclidean Clustering
# TODO: Create Cluster-Mask Point Cloud to visualize each cluster separately
print('Finding object clusters.')
cluster_indices = seg.get_euclidean_cluster_indices(
objects_cloud, 0.03, (10, 5000))
#colorized_object_clusters = seg.return_colorized_clusters(objects_cloud, cluster_indices, self.color_list)
detected_objects, detected_objects_dict = seg.detect_objects(
objects_cloud, cluster_indices)
# TODO: Convert PCL data to ROS messages
# TODO: Publish ROS messages
print('Converting PCL data to ROS messages.')
message_pairs = [ #(decimated_cloud, self.reduced_cloud_pub),
(denoised_cloud, self.denoised_pub),
(objects_cloud, self.objects_pub)
#(table_cloud, self.table_pub),
#(colorized_object_clusters, self.colorized_cluster_pub)
#(passthroughy_cloud, self.passthroughy_filter_pub),
#(passthroughz_cloud, self.passthroughz_filter_pub),
]
seg.convert_and_publish(message_pairs)
#publish detected objects and labels
seg.publish_detected_objects(detected_objects, self.object_markers_pub,
self.detected_objects_pub)
self.object_list = detected_objects_dict
self.detected_objects = detected_objects
self.table_cloud = table_cloud
def move_world_joint(self, goal):
print('Attempting to move world joint to {}'.format(goal))
pub_j1 = rospy.Publisher('/pr2/world_joint_controller/command',
Float64,
queue_size=10)
increments = 10
position = self.get_world_joint_state()
goal_positions = [
n * 1.0 / increments * (goal - position) + position
for n in range(1, increments + 1)
]
for i, g in enumerate(goal_positions):
print('Publishing goal {0}: {1}'.format(i, g))
while abs(position - g) > .005:
pub_j1.publish(g)
position = self.get_world_joint_state()
print('Position: {0}, Error: {1}'.format(
position, abs(position - g)))
rospy.sleep(1)
def build_collision_map(self):
# update this to include drop box object cloud as well
# would need to train SVM for this
self.collision_map_base_list.extend(list(self.table_cloud))
def publish_collision_map(self, picked_objects):
obstacle_cloud_list = self.collision_map_base_list
for obj in self.detected_objects:
if obj.label not in picked_objects:
print('Adding {0} to collision map.'.format(obj.label))
obj_cloud = ros_to_pcl(obj.cloud)
obstacle_cloud_list.extend(list(obj_cloud))
#print(obstacle_cloud)
else:
print(
'Not adding {0} to collision map because it appears in the picked object list'
.format(obj.label))
obstacle_cloud = pcl.PointCloud_PointXYZRGB()
obstacle_cloud.from_list(obstacle_cloud_list)
self.segmenter.convert_and_publish([(obstacle_cloud,
self.collision_cloud_pub)])
def find_pick_object(self, obj):
ppd = pick_place_data()
group = obj['group']
name = obj['name']
print('Looking for pick object: {}'.format(name))
#See if object is found within object list
#if so, use position to populate pick_place_data object
pos = self.object_list.get(name)
if pos is not None:
print('Object found!')
# TODO: Get the PointCloud for a given object and obtain it's centroid
ppd.object_name.data = name
ppd.pick_pose_point.x = np.asscalar(pos[0])
ppd.pick_pose_point.y = np.asscalar(pos[1])
ppd.pick_pose_point.z = np.asscalar(pos[2])
ppd.pick_pose.position = ppd.pick_pose_point
# TODO: Create 'place_pose' for the object
dropboxdata = self.dropbox_dict[group]
#randomize x postion for drop to keep objects from piling up
ppd.place_pose_point.x = dropboxdata.pos[0] - uniform(.1, .2)
ppd.place_pose_point.y = dropboxdata.pos[1] + uniform(-.03, .03)
ppd.place_pose_point.z = dropboxdata.pos[2]
ppd.place_pose.position = ppd.place_pose_point
# TODO: Assign the arm to be used for pick_place
ppd.arm_name.data = dropboxdata.arm
print(
"Scene %d, picking up found %s object, using %s arm, and placing it in the %s bin."
% (ppd.test_scene_num.data, ppd.object_name.data,
ppd.arm_name.data, group))
# TODO: Create a list of dictionaries (made with make_yaml_dict())
# for later output to yaml format
yaml_dict = ppd.return_yaml_dict()
self.dict_list.append(yaml_dict)
return ppd
else:
print("Label: %s not found" % name)
return None
def get_pick_object(self, ppd):
# Use pick_place_routine service proxy to get the object
print('Waiting for "pick_place_routine" service...')
rospy.wait_for_service('pick_place_routine')
try:
pick_place_routine = rospy.ServiceProxy('pick_place_routine',
PickPlace)
print('Sending pick and place data and waiting for response')
resp = pick_place_routine(ppd.test_scene_num, ppd.object_name,
ppd.arm_name, ppd.pick_pose,
ppd.place_pose)
print('Response: '.format(resp.success))
except rospy.ServiceException as e:
print('Service call failed: {}'.format(e))
def mover(self):
print('Starting mover method.')
# TODO: Get/Read parameters
object_list_param = rospy.get_param('/object_list')
# Get dropbox parameters
dropbox_param = rospy.get_param('/dropbox')
# TODO: Parse parameters into individual variables
for dropbox in dropbox_param:
dropboxdata = dropbox_data(dropbox['position'], dropbox['name'])
self.dropbox_dict[dropbox['group']] = dropboxdata
#print('Acquired dropbox data: {}'.format(dropboxdata))
#print('self.dropbox_dict: {}'.format(self.dropbox_dict))
# TODO: Loop through the pick list
picked = []
self.success_count = 0
#self.publish_collision_map('', picked)
for obj in object_list_param:
#get ppd object containing pick and place parameters
ppd = self.find_pick_object(obj)
#if successful, request that pick_place_routine service
#publish all other objects + table to collision map
#get the object
if ppd is not None:
#update list of picked objects
picked.append(obj['name'])
self.success_count += 1
print(
'Successfully identified object ({0}) at pick pose point {1}'
.format(obj['name'], ppd.pick_pose_point))
#republish collision map excluding the objects that have been picked
#currently the map is not refreshing each iteration...
self.publish_collision_map(picked)
#commented out because my computer is too slow...
#grab the object and drop in the bin
self.get_pick_object(ppd)
# TODO: Output your request parameters into output yaml file
#if self.max_success_count < self.success_count:
if not self.yaml_saved:
yaml_filename = "./output/output_" + str(
ppd.test_scene_num.data) + ".yaml"
print("output file name = %s" % yaml_filename)
self.send_to_yaml(yaml_filename, self.dict_list)
#self.max_success_count = success_count
self.yaml_saved = True
print("Successfully picked up {0} of {1} objects".format(
self.success_count, len(object_list_param)))
def pcl_callback(self, pcl_msg):
# TODO: Rotate PR2 in place to capture side tables for the collision map
# commented out world joint rotation because it runs very slowly on my machine
# ideally would want to train SVM to recognize boxes, then add them plus the
# table to the base collision map
if len(self.goal_positions) > 0:
new_position = self.goal_positions.pop()
self.move_world_joint(new_position)
#segment scene and detect objects
self.segment_scene(pcl_msg)
#add obstacles (except for recognized pick objects)
#to the base collision map
self.build_collision_map()
else:
#identify the objects listed in the pick list
#submit them to the pick_place_routine
self.mover()
class ClassifierMatrix:
def __init__(self, config, nodeName, loadFromFile=False):
self.node = config.GetChild(nodeName)
self.segmenter = Segmenter(config, "__segmenter__")
self.trained = loadFromFile
PyMining.Init(config, "__global__", loadFromFile)
"""
create train matrix:
fill dict in PyMining, record:
1)termToId
2)idToTerm
3)termToDocCount
4)classToDocCount
and save mat-x using csr, save mat-y using list
"""
def CreateTrainMatrix(self, path=""):
#get input-path
inputPath = path
if (inputPath == ""):
inputPath = self.node.GetChild("train_input").GetValue()
f = open(inputPath, "r")
uid = 0
rows = [0]
cols = []
vals = []
y = []
#fill the matrix's cols and rows
for line in f:
vec = line.split("\t")
line = vec[0]
target = int(vec[1])
y.append(target)
wordList = self.segmenter.Split(line.decode("utf-8"))
#store current row's cols
partCols = []
#create dicts and fill partCol
#calculate term-frequent in this loop
curWordCount = 0
termFres = {}
for word in wordList:
curWordCount += 1
if (not PyMining.termToId.has_key(word)):
PyMining.termToId[word] = uid
PyMining.idToTerm[uid] = word
uid += 1
termId = PyMining.termToId[word]
partCols.append(termId)
if (not termFres.has_key(termId)):
termFres[termId] = 1
else:
termFres[termId] += 1
#fill partCol
partCols = set(partCols)
partCols = list(partCols)
partCols.sort()
#fill cols and vals, fill termToDocCount
for col in partCols:
cols.append(col)
#fill vals with termFrequent
vals.append(termFres[col])
#fill idToDocCount
if (not PyMining.idToDocCount.has_key(col)):
PyMining.idToDocCount[col] = 1
else:
PyMining.idToDocCount[col] += 1
#fill rows
rows.append(rows[len(rows) - 1] + \
len(partCols))
#fill classToDocCount
if (not PyMining.classToDocCount.has_key(target)):
PyMining.classToDocCount[target] = 1
else:
PyMining.classToDocCount[target] += 1
#fill PyMining's idToIdf
for termId in PyMining.idToTerm.keys():
PyMining.idToIdf[termId] = math.log(
float(len(rows) - 1) / (PyMining.idToDocCount[termId] + 1))
#NOTE: now, not mul idf to vals, because not all algorithms need tf * idf
#change matrix's vals using tf-idf represent
#for r in range(len(rows) - 1):
# for c in range(rows[r], rows[r + 1]):
# termId = cols[c]
# #idf(i) = log(|D| / |{d (ti included)}| + 1
# vals[c] = vals[c] * PyMining.idToIdf[termId]
#close file
f.close()
#write dicts out
PyMining.Write()
self.trained = True
return [Matrix(rows, cols, vals), y]
def CreatePredictSample(self, src):
print src
if (not self.trained):
print "train Classifier Matrix before predict"
#split sentence
#if src is read from utf-8 file directly,
# should using CreatePredictSample(src.decode("utf-8"))
wordList = self.segmenter.Split(src)
cols = []
vals = []
#fill partCols, and create csr
partCols = []
termFreqs = {}
for word in wordList:
if (PyMining.termToId.has_key(word)):
termId = PyMining.termToId[word]
partCols.append(termId)
if (not termFreqs.has_key(termId)):
termFreqs[termId] = 1
else:
termFreqs[termId] += 1
partCols = set(partCols)
partCols = list(partCols)
partCols.sort()
for col in partCols:
cols.append(col)
vals.append(termFreqs[col])
return [cols, vals]
"""
create predict matrix using previous dict
"""
def CreatePredictMatrix(self, path=""):
if (not self.trained):
print "train ClassifierMatrix before predict"
return False
#get input path
inputPath = path
if (inputPath == ""):
inputPath = self.curNode.GetChild("test_input")
f = open(inputPath, "r")
rows = [0]
cols = []
vals = []
y = []
for line in f:
vec = line.split("\t")
line = vec[0]
y.append(int(vec[1]))
#split sentence
wordList = self.segmenter.Split(line.decode("utf-8"))
#fill partCols, and create csr
partCols = []
termFreqs = {}
curWordCount = 0
for word in wordList:
curWordCount += 1
if (PyMining.termToId.has_key(word)):
termId = PyMining.termToId[word]
partCols.append(termId)
if (not termFreqs.has_key(termId)):
termFreqs[termId] = 1
else:
termFreqs[termId] += 1
partCols = set(partCols)
partCols = list(partCols)
partCols.sort()
for col in partCols:
cols.append(col)
vals.append(termFreqs[col])
rows.append(rows[len(rows) - 1] + \
len(partCols))
#close file
f.close()
return [Matrix(rows, cols, vals), y]
def optimize(avg_speed=130, length=1055):
jams = Jam_Simulator()
segmenter = Segmenter()
segmenter = segmenter.get_segmented_route(
length, jams.generate_traffic_jams(route_length=length))
predictor = Predictor()
print(segmenter)
jams = []
for segment in segmenter:
if not segment.jam_info == None:
jam = segment.jam_info
jam.number = segment.number
jams.append(jam)
jam.end = round(
predictor.predict(
np.array([
jam.length, jam.avg_speed, jam.reason, jam.start_time
]).reshape((1, 4)))[0][0], 2)
print(segment.number)
print("******")
for jam in reversed(jams):
strecke = segmenter[jam.number].start
avg_speed = min(avg_speed, round(strecke / jam.end, 2))
if avg_speed < 40:
stri = str("roads are bad. stay at home.")
f = open("../templates/tmp.txt", "w")
f.write(stri)
f.close()
return "roads are bad. stay at home."
for i in range(jam.number):
(segmenter[i]).avg_speed = avg_speed
time = 0
benzin = 0
strom = 0
stri = str()
stri += "length, speed\n"
for segment in segmenter:
print(segment.number)
print(round(segment.length, 2))
print()
stri += "[" + str(round(segment.length, 2)) + ", " + str(
round(segment.avg_speed, 2)) + "]\n"
time += segment.length / segment.avg_speed
benzin += (3.4 + 0.00966667 * segment.avg_speed - 0.00024 * (segment.avg_speed**2) \
+ 2.53333*(10**(-6))*(segment.avg_speed**3)) * (segment.length * 0.01)
strom += (-1.1 + 0.06 * segment.avg_speed - 0.00025 *
(segment.avg_speed**2)) * (segment.length * 0.01)
print("time:")
print(round(time, 2))
print("benzin")
print(round(benzin, 2))
print("strom")
print(round(strom, 2))
stri += "Time: " + str(round(time, 2)) + "\n"
stri += "Benzin: " + str(round(benzin, 2)) + "\n"
stri += "Strom: " + str(round(strom, 2)) + "\n"
f = open("../templates/tmp.txt", "w")
f.write(stri)
f.close()
def main():
# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-m", "--draw-markers", action="store_true", default=False,
help="")
ap.add_argument("-c", "--draw-confidence", action="store_true", default=False,
help="")
ap.add_argument("-t", "--confidence-threshold", type=float, default=0.9,
help="")
ap.add_argument("-p", "--draw-pose", action="store_false", default=True,
help="")
ap.add_argument("-u", "--draw-unstable", action="store_true", default=False,
help="")
ap.add_argument("-s", "--draw-segmented", action="store_true", default=False,
help="")
args = vars(ap.parse_args())
confidence_threshold = args["confidence_threshold"]
"""MAIN"""
# Video source from webcam or video file.
video_src = 0
cam = cv2.VideoCapture(video_src)
_, sample_frame = cam.read()
# Introduce mark_detector to detect landmarks.
mark_detector = MarkDetector()
# Setup process and queues for multiprocessing.
img_queue = Queue()
box_queue = Queue()
img_queue.put(sample_frame)
if isWindows():
thread = threading.Thread(target=get_face, args=(mark_detector, confidence_threshold, img_queue, box_queue))
thread.daemon = True
thread.start()
else:
box_process = Process(target=get_face,
args=(mark_detector, confidence_threshold, img_queue, box_queue))
box_process.start()
# Introduce pose estimator to solve pose. Get one frame to setup the
# estimator according to the image size.
height, width = sample_frame.shape[:2]
pose_estimator = PoseEstimator(img_size=(height, width))
# Introduce scalar stabilizers for pose.
pose_stabilizers = [Stabilizer(
state_num=2,
measure_num=1,
cov_process=0.1,
cov_measure=0.1) for _ in range(6)]
while True:
# Read frame, crop it, flip it, suits your needs.
frame_got, frame = cam.read()
if frame_got is False:
break
# Crop it if frame is larger than expected.
# frame = frame[0:480, 300:940]
# If frame comes from webcam, flip it so it looks like a mirror.
if video_src == 0:
frame = cv2.flip(frame, 2)
# Pose estimation by 3 steps:
# 1. detect face;
# 2. detect landmarks;
# 3. estimate pose
# Feed frame to image queue.
img_queue.put(frame)
# Get face from box queue.
result = box_queue.get()
if result is not None:
if args["draw_confidence"]:
mark_detector.face_detector.draw_result(frame, result)
# unpack result
facebox, confidence = result
# fix facebox if needed
if facebox[1] > facebox[3]:
facebox[1] = 0
if facebox[0] > facebox[2]:
facebox[0] = 0
# Detect landmarks from image of 128x128.
face_img = frame[facebox[1]: facebox[3],
facebox[0]: facebox[2]]
face_img = cv2.resize(face_img, (CNN_INPUT_SIZE, CNN_INPUT_SIZE))
face_img = cv2.cvtColor(face_img, cv2.COLOR_BGR2RGB)
marks = mark_detector.detect_marks(face_img)
# Convert the marks locations from local CNN to global image.
marks *= (facebox[2] - facebox[0])
marks[:, 0] += facebox[0]
marks[:, 1] += facebox[1]
# segment the image based on markers and facebox
seg = Segmenter(facebox, marks, frame.shape[1], frame.shape[0])
if args["draw_segmented"]:
mark_detector.draw_box(frame, seg.getSegmentBBs())
cv2.imshow("fg", seg.getSegmentJSON()["faceGrid"])
if args["draw_markers"]:
mark_detector.draw_marks(
frame, marks, color=(0, 255, 0))
# Try pose estimation with 68 points.
pose = pose_estimator.solve_pose_by_68_points(marks)
# Stabilize the pose.
stable_pose = []
pose_np = np.array(pose).flatten()
for value, ps_stb in zip(pose_np, pose_stabilizers):
ps_stb.update([value])
stable_pose.append(ps_stb.state[0])
stable_pose = np.reshape(stable_pose, (-1, 3))
if args["draw_unstable"]:
pose_estimator.draw_annotation_box(
frame, pose[0], pose[1], color=(255, 128, 128))
if args["draw_pose"]:
pose_estimator.draw_annotation_box(
frame, stable_pose[0], stable_pose[1], color=(128, 255, 128))
# Show preview.
cv2.imshow("Preview", frame)
if cv2.waitKey(10) == 27:
break
# Clean up the multiprocessing process.
if not isWindows():
box_process.terminate()
box_process.join()
from segmenter import Segmenter
import argparse
from model.yaml_config import parse_options
if __name__ == '__main__':
args = argparse.ArgumentParser()
args.add_argument('--config_file', '-c', default=None, type=str)
args.add_argument('--embedding_dim', default=100, type=int)
args.add_argument('--hidden_dim', type=int, default=100)
args.add_argument('--clf_hidden_dim', type=int, default=100)
args.add_argument('--epoch', type=int, default=20)
args.add_argument('--batch_size', type=int, default=32)
args.add_argument('--lstm_layer', type=int, default=2)
args.add_argument('--dropout', type=float, default=0.5)
args.add_argument('--model_pth', type=str)
args.add_argument('--env', '-e', default='train', type=str)
args.add_argument('--test_target', type=str)
args.add_argument('--data_dir', type=str)
options = parse_options(args)
if options.train:
Segmenter.train(options)
elif options.test:
fscore = Segmenter.test(options.model_pth, options.test_path)
print('Test fscore :{}'.format(fscore))
elif options.seg:
Segmenter.seg_corpus(options.model_pth, options.raw_corpus,
options.output)
def __init__(self, config, nodeName, loadFromFile=False):
self.node = config.GetChild(nodeName)
self.segmenter = Segmenter(config, "__segmenter__")
self.trained = loadFromFile
PyMining.Init(config, "__global__", loadFromFile)
from neuralNetworkTester import NeuralNetworkTester
from remover import Remover
import subprocess
import config
import utils
from datetime import datetime
from logger import Logger
input_file = open(config.CATEGORY_LIST, "r")
categories = input_file.read().splitlines()
input_file.close()
generator = LinkGen()
generator.run(categories)
downloader = Downloader(categories, config.BATCH_SIZE)
segmenter = Segmenter()
remover = Remover()
logger = Logger()
processed_video_count = utils.get_processed_video_count()
while processed_video_count < config.NUMBER_OF_LINKS:
processed_video_count = utils.get_processed_video_count()
load = utils.get_checkpoints_flag()
downloader.run()
segmenter.run()
source = config.NEURAL_NETWORK_PATH
tfrecords_command = 'python2.7 %s/utils/generate_tfrecords_dataset.py' \
