Esempio n. 1
0
    def get_page(self, page):
        myUrl = 'http://android.myapp.com/myapp/app/comment.htm?apkName=com.msxf.loan&apkCode=15701&p=' + page + '&contextData=' + self.contextData
        user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
        headers = {'User-Agent': user_agent}
        req = urllib.request.Request(myUrl, headers=headers)
        myResponse = urllib.request.urlopen(req)
        myPage = myResponse.read()
        # encode的作用是将unicode编码转换成其他编码的字符串
        # decode的作用是将其他编码的字符串转换成unicode编码
        unicodePage = myPage.decode("utf-8")

        jsondata = json.loads(unicodePage)
        if (not jsondata == None) and 'obj' in jsondata:
            obj = jsondata['obj']
            if not obj == None:
                if self.total == 0:
                    if 'total' in obj:
                        self.total = obj['total']
                if 'commentDetails' in obj:
                    commentDetailes = obj['commentDetails']
                if 'contextData' in obj:
                    self.contextData = obj['contextData']

                self.crawlCount += len(commentDetailes)
                reviews = []
                for comment in commentDetailes:
                    review = Review()
                    review.appStore = 'myapp'
                    review.packageName = 'com.msxf.loan'

                    if 'content' in comment:
                        review.content = comment['content']
                    if 'nickName' in comment:
                        review.nickName = comment['nickName']
                    if 'score' in comment:
                        review.score = comment['score']
                    if 'versionCode' in comment:
                        review.versionCode = comment['versionCode']
                    if 'createdTime' in comment:
                        review.reviewTime = datetime.datetime.fromtimestamp(int(comment['createdTime'])).strftime(
                            '%Y-%m-%d %H:%M:%S')
                    reviews.append(review)
                ReviewsDataSource.insert(reviews)

        self.enable = self.crawlCount < self.total
Esempio n. 2
0
def queryTodayReviews(reviews):
    conn = sqlite3.connect('reviews.db')

    try:
        c = conn.cursor()
        for row in c.execute(
                "select nickName,content,reviewTime,appStore,versionCode,packageName,score from reviews WHERE date(reviews.reviewTime) = date('now')"):
            review = Review()
            review.appStore = 'myapp'
            review.nickName = row[0]
            review.content = row[1]
            review.reviewTime = row[2]
            review.appStore = row[3]
            review.versionCode = row[4]
            review.packageName = row[5]
            review.score = row[6]
            reviews.append(review)
    except BaseException as e:
        print('sql error : ' + e.__cause__)

    c.close()
    print('queryTodayReviews ' + str(len(reviews)) + ' row.')