Beispiel #1
0
def normGoogleWall(jresult):
	posts = list()
	page_count = 0
	if type(jresult) == list:
		for page in jresult:
			if page_count > 10:
				# revise the size in the future
				break
			for post in page["items"]:
				published_time = formatGoogleTime(post["published"])
				place = formatGooglePlace(post.get("location", ""), 2)
				info = post.get("object", "")
				if info != "":
					text = info.get("content", "")
					urls = getGoogleUrls(info.get("attachments", ""))
					# a = time.time()
					lang = ut.detectLang(text)
					# b = time.time()
					text_en = ut.translate(text, lang)
					# c= time.time()
					sentiment = ut.getSentiment(text_en)
					# d= time.time()
					topic_distri = ut.getTopic(text_en)
					tf = ut.wordProcess(text, lang)
					# e= time.time()
					# print(b-a, c-b, d-c, e-d)
					posts.append(getPost(text, text_en, published_time, place, urls, lang, sentiment, topic_distri, tf))
			page_count+=1
	return posts
Beispiel #2
0
def normGoogleWall(jresult):
    posts = list()
    page_count = 0
    if type(jresult) == list:
        for page in jresult:
            if page_count > 10:
                # revise the size in the future
                break
            for post in page["items"]:
                published_time = formatGoogleTime(post["published"])
                place = formatGooglePlace(post.get("location", ""), 2)
                info = post.get("object", "")
                if info != "":
                    text = info.get("content", "")
                    urls = getGoogleUrls(info.get("attachments", ""))
                    # a = time.time()
                    lang = ut.detectLang(text)
                    # b = time.time()
                    text_en = ut.translate(text, lang)
                    # c= time.time()
                    sentiment = ut.getSentiment(text_en)
                    # d= time.time()
                    topic_distri = ut.getTopic(text_en)
                    tf = ut.wordProcess(text, lang)
                    # e= time.time()
                    # print(b-a, c-b, d-c, e-d)
                    posts.append(
                        getPost(text, text_en, published_time, place, urls,
                                lang, sentiment, topic_distri, tf))
            page_count += 1
    return posts
Beispiel #3
0
def normTwitterWall(wall):
	posts = list()
	for post in wall:
		text = post.get("text", "")
		time = formatTwitterTime(post.get("created_at"))
		place = formatTwitterPlace(post["geo"], 2)
		urls = getTwitterUrls(post)
		lang = post.get("lang", "")
		if lang == "":
			lang = ut.detectLang(text)
		# translate text
		text_en = ut.translate(text, lang)
		sentiment = ut.getSentiment(text_en)
		topic_distri = ut.getTopic(text_en)
		tf = ut.wordProcess(text, lang)
		posts.append(getPost(text, text_en, time, place, urls, lang, sentiment, topic_distri, tf))
	return posts
Beispiel #4
0
def normTwitterWall(wall):
    posts = list()
    for post in wall:
        text = post.get("text", "")
        time = formatTwitterTime(post.get("created_at"))
        place = formatTwitterPlace(post["geo"], 2)
        urls = getTwitterUrls(post)
        lang = post.get("lang", "")
        if lang == "":
            lang = ut.detectLang(text)
        # translate text
        text_en = ut.translate(text, lang)
        sentiment = ut.getSentiment(text_en)
        topic_distri = ut.getTopic(text_en)
        tf = ut.wordProcess(text, lang)
        posts.append(
            getPost(text, text_en, time, place, urls, lang, sentiment,
                    topic_distri, tf))
    return posts
def augmentation(imgPath, name):
    # generateFunc = ["original", "mirror", "rotate", "translate", "brightnessAndContrast"]
    # generateFunc = ["mirror", "rotate", "translate", "brightnessAndContrast", "blur"]
    generateFunc = ["mirror", "brightnessAndContrast", "blur"]

    if debug:
        print "imgPath + name: ", imgPath + name

    img = cv2.imread(imgPath + name)
    x, y = [], []

    img = ut.resize(img)
    newImg, x, y = ut.scale(img, [], [], imSize=224)
    print "name: ", name
    print "type(img): ", type(img)
    print "img.shape: ", img.shape
    if img != None:
        if debug:
            print "FIND image: ", imgPath + name
            print "img.shape: ", img.shape
        derivateNum = len(generateFunc)
        for index in range(derivateNum):

            (w, h, _) = img.shape
            # method = random.choice(generateFunc)
            method = generateFunc[index]

            # if index == 0:
            # 	method = "original"

            # if method == "resize":
            #     newImg, newX, newY = ut.resize(img, x, y, xMaxBound = w, yMaxBound = h, random = True)
            if method == "rotate":
                newImg, newX, newY = ut.rotate(img, x, y, w=w, h=h)
            elif method == "mirror":
                newImg, newX, newY = ut.mirror(img, x, y, w=w, h=h)
            elif method == "translate":
                newImg, newX, newY = ut.translate(img, x, y, w=w, h=h)
            elif method == "brightnessAndContrast":
                newImg, newX, newY = ut.contrastBrightess(img, x, y)
            elif method == "original":
                newImg, newX, newY = img, x, y
            elif method == "blur":
                newImg, newX, newY = blur = cv2.blur(img, (5, 5)), x, y
                # newImg, newX, newY = ut.scale(img, x, y, imSize = imSize)
            # elif method == "scale":
            # 	newImg, newX, newY = ut.scale(img, x, y)
            else:
                raise "not existing function"

            if debug:
                print "name: ", name
                print "index: ", index
                print "newImg.shape: ", newImg.shape
                print "method: ", method
                cv2.imshow("img", img)
                cv2.imshow("newImg", newImg)
                cv2.waitKey(0)

            cv2.imwrite(
                path + name.replace(".jpg", "") + "_" + str(index) + ".jpg",
                newImg)
            print "saving to ............"
            print path + name.replace(".jpg", "") + "_" + str(index) + ".jpg"
Beispiel #6
0
    def DataGenBB(self, DataStrs, train_start, train_end):
        generateFunc = [
            "original", "scale", "rotate", "translate", "scaleAndTranslate",
            "brightnessAndContrast"
        ]
        # generateFunc = ["original", "scale", "rotate", "translate", "scaleAndTranslate"]

        InputData = np.zeros(
            [self.batch_size * len(generateFunc), self.imSize, self.imSize, 3],
            dtype=np.float32)
        # InputLabel = np.zeros([self.batch_size * len(generateFunc), 7], dtype = np.float32)
        InputLabel = np.zeros([self.batch_size * len(generateFunc), 3],
                              dtype=np.float32)

        InputNames = []
        count = 0
        for i in range(train_start, train_end):
            strLine = DataStrs[i]
            strCells = strLine.rstrip(' \n').split(' ')
            imgName = strCells[0]

            labels = np.array(strCells[1:]).astype(np.float)

            if len(labels) == 78:
                # print "switch to menpo39"
                labelsPTS = labels[:136].reshape([39, 2])
                self.ifMenpo39Data = True
            else:
                # print "not menpo39"
                labelsPTS = labels[:136].reshape([68, 2])
                self.ifMenpo39Data = False

            # if self.debug:
            #     print "imgName: ", imgName
            img = cv2.imread(imgName)

            if img != None:
                # print "find image: ", imgName
                # print "img.shape: ", img.shape

                img = cv2.resize(img, (self.imSize, self.imSize))
                # print "img.shape: ", img.shape
                if self.ifMenpo39Data:
                    x, y = self.unpackLandmarks(labelsPTS)
                else:
                    x, y = ut.unpackLandmarks(labelsPTS, self.imSize)

                # newImg, newX, newY = img, x, y

                for index in range(len(generateFunc)):
                    method = generateFunc[index]
                    (w, h, _) = img.shape
                    # tag = random.choice(generateFunc)
                    if method == "resize":
                        newImg, newX, newY = ut.resize(img,
                                                       x,
                                                       y,
                                                       xMaxBound=w,
                                                       yMaxBound=h,
                                                       random=True)
                    elif method == "rotate":
                        newImg, newX, newY = ut.rotate(img, x, y, w=w, h=h)
                    elif method == "mirror":
                        newImg, newX, newY = ut.mirror(img, x, y, w=w, h=h)
                    elif method == "translate" or method == "scaleAndTranslate":
                        newImg, newX, newY = ut.translate(img, x, y, w=w, h=h)
                    elif method == "brightnessAndContrast":
                        newImg, newX, newY = ut.contrastBrightess(img, x, y)
                    elif method == "original":
                        newImg, newX, newY = img, x, y
                    elif method == "scale":
                        newImg, newX, newY = img, x, y
                    else:
                        raise "not existing function"

                    # if self.debug:
                    #     plotOriginal = ut.plotLandmarks(img, x, y, self.imSize, ifReturn = True)
                    #     plotNew = ut.plotLandmarks(newImg, newX, newY, self.imSize, ifReturn = True)

                    #     cv2.imwrite(self.outputDir + 'testOriginal' + str(count) + '.jpg', img)
                    #     cv2.imwrite(self.outputDir + 'testNew' + str(count) + '.jpg', newImg)
                    #     cv2.imwrite(self.outputDir + 'plotOriginal' + str(count) + '.jpg', plotOriginal)
                    #     cv2.imwrite(self.outputDir + 'plotNew' + str(count) + '.jpg', plotNew)

                    # print "before normalize: ", newX

                    # normX = ut.normalize(newX)
                    # normY = ut.normalize(newY)

                    # print "after normalize: ", newX
                    # print "after denormalize again: ", ut.deNormalize(newX)

                    # normXMin = min(normX)
                    # normYMin = min(normY)
                    # normXMax = max(normX)
                    # normYMax = max(normY)
                    # normXMean = (normXMax + normXMin)/2.0
                    # normYMean = (normYMax + normYMin)/2.0
                    # normEdge = max(normYMax - normYMin, normXMax - normXMin)
                    newXMin = min(newX)
                    newYMin = min(newY)
                    newXMax = max(newX)
                    newYMax = max(newY)
                    newXMean = (newXMax + newXMin) / 2.0
                    newYMean = (newYMax + newYMin) / 2.0
                    edge = max(newYMax - newYMin, newXMax - newXMin)

                    # if method == "scale":
                    #     cv2.imshow("originalImg", newImg)
                    #     cv2.waitKey(0)

                    if method == "scale" or method == "scaleAndTranslate":
                        newEdge = np.random.uniform(0.7, 0.9) * edge
                        newXMin = int(newXMean - newEdge / 2.0)
                        newXMax = int(newXMean + newEdge / 2.0)
                        newYMin = int(newYMean - newEdge / 2.0)
                        newYMax = int(newYMean + newEdge / 2.0)

                        newXMean = newXMean - newXMin
                        newYMean = newYMean - newYMin

                        # print "newXMin, newYMin, newXMax, newYMax: ", newXMin, newYMin, newXMax, newYMax

                        newImg = Image.fromarray(newImg.astype(np.uint8))
                        cropImg = newImg.crop(
                            (newXMin, newYMin, newXMax, newYMax))
                        newImg = np.array(cropImg)

                        # cv2.imshow("processing", newImg)
                        # cv2.waitKey(0)

                        w, h, _ = newImg.shape
                        edge = edge * self.imSize / w
                        newXMean = newXMean * self.imSize / w
                        newYMean = newYMean * self.imSize / h
                        newImg = cv2.resize(newImg, (self.imSize, self.imSize))

                    # print "newXMin: ", newXMin
                    # print "newYMin: ", newYMin
                    # print "newXMax: ", newXMax
                    # print "newYMax: ", newYMax
                    # print "newXMean: ", newXMean
                    # print "newYMean: ", newYMean
                    # print "newEdge: ", newEdge

                    # if method == "scale":
                    #     newImg = ut.plotTarget(newImg, [newXMean, newYMean, edge], ifSquareOnly = True, ifGreen = True)
                    #     cv2.imshow("newImg", newImg)
                    #     cv2.waitKey(0)

                    # if self.ifMenpo39Data == False:
                    print "imgName: ", imgName.split("/")[-2] + imgName.split(
                        "/")[-1].split(".")[0]
                    print "inputCheck/" + imgName.split(
                        "/")[-2] + imgName.split("/")[-1].split(".")[0] + str(
                            method) + str(count) + '.jpg'
                    cv2.imwrite(
                        "inputCheck/" + imgName.split("/")[-2] +
                        imgName.split("/")[-1].split(".")[0] + str(method) +
                        str(count) + '.jpg', newImg)

                    normX = ut.normalize(newX, self.imSize)
                    normY = ut.normalize(newY, self.imSize)
                    # normPTS = np.asarray(ut.packLandmarks(normX, normY))
                    normXMean, normYMean, normEdge = ut.normalize(
                        newXMean, self.imSize), ut.normalize(
                            newYMean,
                            self.imSize), ut.normalize(edge, self.imSize)
                    # print "newPTS: ", newPTS.shape

                    # print "ut.deNormalize(normXMin): ", ut.deNormalize(normXMin)
                    # print "ut.deNormalize(normYMin): ", ut.deNormalize(normYMin)
                    # print "ut.deNormalize(normXMax): ", ut.deNormalize(normXMax)
                    # print "ut.deNormalize(normYMax): ", ut.deNormalize(normYMax)
                    # print "ut.deNormalize(normXMean): ",ut.deNormalize(normXMean)
                    # print "ut.deNormalize(normYMean): ",ut.deNormalize(normYMean)
                    # print "ut.deNormalize(normEdge): ", ut.deNormalize(normEdge)

                    # print "method: ", method
                    # print "newImg.shape: ", newImg.shape

                    # print "len(InputData): ", len(InputData)
                    InputData[count, ...] = newImg
                    # labels = np.array([normPTS[27][0], normPTS[27][1], normPTS[8][0],
                    #     normPTS[8][1], normXMean, normYMean, normEdge])
                    labels = np.array([normXMean, normYMean, normEdge])
                    InputLabel[count, ...] = labels
                    InputNames.append(imgName)

                    # print "count: ", count
                    count += 1

            else:
                print "cannot : ", imgName

        return InputData, InputLabel, np.asarray(InputNames)
 def pixel_y(self):
     return int(translate(self.y, -1, 1, self.height, 0))
 def pixel_x(self):
     return int(translate(self.x, -1, 1, 0, self.width))