예제 #1
0
def getYesterdayKDJ(rlist):
	cyStockData = Object.extend('CYStockData')
	query = Query(cyStockData)  
	query.equal_to('kdjback', 1)
	query.greater_than('kdj',0)
	query.equal_to('volume',1)
	cyresults = query.find()
	for x in cyresults:		
		rlist.append(x.get("stcode"))
	szStockData = Object.extend('SZStockData')
	query = Query(szStockData)  
	query.equal_to('kdjback', 1)
	query.greater_than('kdj',0)
	query.equal_to('volume',1)
	szresults = query.find()
	for x in szresults:
		rlist.append(x.get("stcode"))
	shStockData = Object.extend('SHStockData')
	query = Query(shStockData)  
	query.equal_to('kdjback', 1)
	query.greater_than('kdj',0)
	query.equal_to('volume',1)
	shresults = query.find()
	for x in shresults:
		rlist.append(x.get("stcode"))
def create_demo_application(table_name='DemoApplication'):
    all_real_app = get_all_real_applications()
    DbTable = Object.extend(table_name)

    for index, app in enumerate(all_real_app):
        db_table = DbTable()
        relation = app.relation('tracker')
        # query = Query(DbTable)
        query = relation.query()
        user_list = query.find()
        relation = db_table.relation('tracker')
        for user in user_list:
            ADbTable = Object.extend('Tracker')
            query = Query(ADbTable)
            query.equal_to('objectId',user.id)
            result_list = query.find()
            if result_list:
                tracker = result_list[0]
                relation.add(tracker)
            else:
                print 'tracker not exists and objectId is: %s' %(str(user.id))

        db_table.set('app_name',app.get('app_name'))
        db_table.set('origin_name',app.get('name'))
        db_table.save()
예제 #3
0
파일: web.py 프로젝트: iforgotid/webmail
 def post(self):
     self.response.headers['Content-Type'] = 'application/json'
     data = json.loads(self.request.body)
     Mail = Object.extend('Mail')
     newMail = Mail()
     newMail.set('subject',data['subject'])
     newMail.set('to',data['to'])
     newMail.set('html',data['html'])
     newMail.save()
     type = data['type']
     Timer = Object.extend('Timer')
     timer = Timer()
     timer.set('mailId',newMail.id)
     timer.set('status','unsent')
     if type == 'byInterval':
         now = datetime.datetime.now()
         timeUnit = data['timeUnit']
         intervalCount = data['intervalCount']
         seconds = 0;
         if timeUnit == 'day':
             seconds = intervalCount * 86400
         elif timeUnit == 'hour':
             seconds = intervalCount * 3600
         elif timeUnit == 'minute':
             seconds = intervalCount * 60
         timeAfter = now + datetime.timedelta(seconds = seconds)
         timestamp = int(time.mktime(timeAfter.timetuple()))
     elif type == 'byTime':
         timeStr = data['timeStr']
         timeArray = time.strptime(timeStr, "%Y-%m-%d %H:%M")
         timestamp = int(time.mktime(timeArray))
     timer.set('timestamp',timestamp)
     timer.save()
     self.response.out.write(json.dumps({'code':0}))
예제 #4
0
 def consume(self):
     queue = SQS()
     rs = queue.read()
     print 'Start consuming:'
     for record in rs:
         mailId = record.get_body()
         print mailId
         Timer = Object.extend('Timer')
         timerQuery = Query(Timer)
         timerQuery.equal_to('mailId', mailId)
         firstTimer = timerQuery.first()
         if firstTimer.get('status') != 'sent':
             Mail = Object.extend('Mail')
             query = Query(Mail)
             mailObj = query.get(mailId)
             sender = mailer()
             mailToSent = {
                 'to':mailObj.get('to'),
                 'html':mailObj.get('html'),
                 'subject':mailObj.get('subject')
             }
             sender.send(mailToSent)
             firstTimer.set('status','sent')
             firstTimer.save()
         queue.remove(record)
     return self;
예제 #5
0
def test_relation(): # type: () -> None
    foo = Object.extend('Foo')()
    foo.set('a', 1)
    bar = Object.extend('Bar')()
    bar.set('baz', 'baz')
    bar.save()
    relation = foo.relation('list')
    relation.add(bar)
    foo.save()
예제 #6
0
def test_relation():  # type: () -> None
    foo = Object.extend("Foo")()
    foo.set("a", 1)
    bar = Object.extend("Bar")()
    bar.set("baz", "baz")
    bar.save()
    relation = foo.relation("list")
    relation.add(bar)
    foo.save()
예제 #7
0
def test_relation():
    foo = Object.extend('Foo')()
    foo.set('a', 1)
    bar = Object.extend('Bar')()
    bar.set('baz', 'baz')
    bar.save()
    relation = foo.relation('list')
    relation.add(bar)
    foo.save()
예제 #8
0
def test_relation():
    foo = Object.extend("Foo")()
    foo.set("a", 1)
    bar = Object.extend("Bar")()
    bar.set("baz", "baz")
    bar.save()
    relation = foo.relation("list")
    relation.add(bar)
    foo.save()
예제 #9
0
def updata_backend_info(parse_dict):
    print parse_dict
    user_id = parse_dict['user_id']
    # get user Object
    query = Query(Object.extend('_User'))
    query.equal_to('objectId', user_id)
    user = query.find()[0] if query.count() else None

    # get app Object
    query = Query(Object.extend('BindingInstallation'))
    query.equal_to('user', user)
    result_list = query.find()
    app_set = set()
    for result in result_list:
        app_set.add(result.attributes['application'].id)
    app_id_list = list(app_set)

    for app_id in app_id_list:
        query = Query(Object.extend('Application'))
        query.equal_to('objectId', app_id)
        app = query.find()[0]

        table_dash = Object.extend('DashboardSource')
        query = Query(table_dash)
        query.equal_to('app', app)
        query.equal_to('user', user)
        dst_table = query.find()
        if not dst_table:
            dst_table = table_dash()
        else:
            dst_table = dst_table[0]

        dst_table.set('app', app)
        for key, value in parse_dict.items():
            if key is 'user_id':
                dst_table.set('user', user)
            elif key is 'home_office_status':
                home_office_status = dst_table.get('home_office_status') or {}
                for k, v in parse_dict['home_office_status'].items():
                    home_office_status[k] = v
                dst_table.set('home_office_status', home_office_status)
            elif key is 'event':
                event = dst_table.get('event') or {}
                for k, v in parse_dict['event'].items():
                    event[k] = v
                dst_table.set('event', event)
            elif key is 'location':
                location = dst_table.get('location') or {}
                for k, v in parse_dict['location'].items():
                    location[k] = v
                dst_table.set('location', location)
            else:
                dst_table.set(key, value)

        dst_table.save()
        return True
def associate_event_and_activity(db_name='MergedUserContext'):

    application_event_dict = get_all_event()
    print 'already get the application_event_dict'
    print str(len(application_event_dict.keys()))
    DBTable = Object.extend(db_name)

    # print 'event_list: %s' %str(application_event_dict.values())
    for application,event_dict in application_event_dict.items():
        # print event_dict
        if event_dict:
            print 'application_event_dict values first count: %s' %str(event_dict.keys())

        EventActivity = Object.extend('FakeEventActivity')
        for event_name,event_list in event_dict.items():
            total_count = len(event_list)
            print 'event_list total_count: %s with event_name is: %s' %(str(total_count),event_name)
            print 'application id is: %s' %str(application.id)

            event_activity = EventActivity()
            relation = event_activity.relation('event')
            activity_dict = {}
            for index,event in enumerate(event_list):
                relation.add(event)
                query = Query(DBTable)
                query.equal_to('tracker',event.get('tracker'))
                query.less_than_or_equal_to('startTime',event.get('timestamp'))
                query.greater_than_or_equal_to('endTime',event.get('timestamp'))
                activity_list = query.find()
                if len(activity_list) == 1 or len(activity_list) == 2 :
                    # for the convenience of adding the dimension of time to the analyzer
                    event.set('activity',activity_list[0])
                    event.save()
                    # activity = activity_list[0].get('eventType')[0]
                    activity = activity_list[0].get('eventType')[0]

                    if activity in activity_dict.keys():
                        activity_dict[activity]+=1
                    else:
                        activity_dict[activity] =1
                else:
                    event.destroy()
                    print 'length of activity_list: %s' %(str(len(activity_list)))
                    print 'Seems to be an error,index: %s,user: %s; timestamp: %s \n' %(str(index),str(event.get('tracker').id ),str(event.get('timestamp')))

            other_activity_total_count =total_count-sum(activity_dict.values())
            if other_activity_total_count:
                activity_dict['others'] = other_activity_total_count

            # EventActivity = Object.extend('EventActivity')
            # event_activity = EventActivity()
            event_activity.set('application',application)
            event_activity.set('event_name',event_list[0].get('event_name'))
            event_activity.set('activity_dict',activity_dict)
            event_activity.save()
예제 #11
0
    def get_age_and_gender_data_dict(self, table_name="AppStaticInfo", filed_name="app"):
        try:

            Application = Object.extend("Application")
            query = Query(Application)
            query.equal_to("app_id", self.app_id)
            result_list = query.find()
            length = len(result_list)
            if length == 0:
                print "error: application not exists in table Applicaiton"
                return 0
            elif length != 1:
                print "error: multi application exists in table Applicaiton"
                return 0
            else:
                app = result_list[0]
                DbTable = Object.extend(table_name)
                query = Query(DbTable)
                query.equal_to(filed_name, app)
                result_list = query.find()
                length = len(result_list)
                if length == 0:
                    print "error: application not exists in table %s" % (str(table_name))
                    return 0
                elif length != 1:
                    print "error: multi application  exists in table %s" % (str(table_name))
                    return 0
                else:
                    app_static_info = result_list[0]
                    age_and_gender_dict = app_static_info.get("age_and_gender")
                    return age_and_gender_dict

            # WeightedStaticInfo  = Object.extend('WeightedStaticInfo')
            # query = Query(WeightedStaticInfo)
            # query.exists('objectId')
            # query.select('age','gender')
            # staticInfoList = query.find()
            # gender_type_list =['man','woman']
            # age_type_list = ['16down','16to35','35to55','55up']
            # dataDict ={gender_type:{age_type:0 for age_type in age_type_list} for gender_type in gender_type_list}
            #
            # for staticInfo in staticInfoList:
            #     gender = 'man' if staticInfo.get('gender') >0 else 'woman'
            #     age_info_dict= staticInfo.get('age')
            #     dataDict[gender][age_info_dict.keys()[0]] += 1
            # # dataDict ={'man' if staticInfo.get('gender') >0 else 'woman':dataDict['man' if staticInfo.get('gender') >0 else 'woman'][staticInfo.get('age').keys()[0]] +=1 for staticInfo in staticInfoList}
            # new_data_dict = {key:[0 for i in range(4)] for key in dataDict.keys()}
            # for index ,age_type in enumerate(age_type_list):
            #     for gender_type in dataDict.keys():
            #         new_data_dict[gender_type][index] = dataDict[gender_type][age_type]

        except LeanCloudError, e:

            raise e
예제 #12
0
    def process_item(self, item, spider):

        Answers = Object.extend('Answers')
        Users = Object.extend('Users')


        for index ,ques in enumerate(item['floorNumList']):
            answer = Answers()
            user = Users()
            queryAnswer = Query(Answers)
            queryUser = Query(Users)
            queryAnswer.equal_to('questionLink',item['questionLink'])
            queryAnswer.equal_to('userId',item['userIdList'][index])
            queryAnswer.equal_to('answerTime',item['answerTimeList'][index])
            try:
                if queryAnswer.find():
                    pass
                else:
                    answer.set('questionLink',item['questionLink'])
                    answer.set('answerPageNum',item['answerPageNum'])

                    answer.set('floorNum',item['floorNumList'][index])
                    answer.set('answerPosition',item['answerPositionList'][index])

                    answer.set('userId',item['userIdList'][index])
                    answer.set('answerTime',item['answerTimeList'][index])
                    answer.set('answerIp',item['answerIpList'][index])
                    answer.set('answerContent',item['answerContentList'][index])
                    try:
                        answer.save()

                    except LeanCloudError,e:
                        print e
            except LeanCloudError,e:
                print e

            queryUser.equal_to('userId',item['userIdList'][index])
            try:
                if queryUser.find():
                    pass
                else:
                    user.set('userId',item['userIdList'][index])
                    user.set('userImgLink',item['userImgLinkList'][index])
                    user.set('userName',item['userNameList'][index])
                    user.set('userClass',item['userClass'][index])
                    user.set('userQuestionCount',item['userQuestionCountList'][index])
                    user.set('userScore',item['userScore'][index])
                    try:
                        user.save()
                    except LeanCloudError,e:
                        print e
            except LeanCloudError,e:
                print e
예제 #13
0
def parseDetailPage(url, word):
    # url = 'http://www.hujiang.com/ciku/noresult/'
    try:
        print url
        global category
        global type_code
        headers = {
            'User-Agent':
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
        }
        req = requests.get(url, headers=headers)
        req.encoding = 'utf-8'
        content = req.text.replace('<br>', '\n')
        content = content.replace('<br />', '\n')
        content = content.replace('<br/>', '\n')
        soup = BeautifulSoup(content, "html5lib")
        article_content = soup.find('div', id="article_content")
        article_title = soup.find('h1', id='article_title')
        if article_title and article_content:
            title = article_title.text.strip()
            content = getContent(article_content.text.strip(), title)
            if isExit(title):
                print 'exit and return'
                return
            else:
                print word
                print title
                print content
                FilmDetail = Object.extend('HJWordStudyCList')
                mFilmDetail = FilmDetail()
                mFilmDetail.set('title', title)
                mFilmDetail.set('word', word)
                mFilmDetail.set('word_des', content)
                mFilmDetail.set('category', category)
                mFilmDetail.set('type', type_code)
                mFilmDetail.set('source_url', url)
                mFilmDetail.save()
                print('save item')
        else:
            print 'has no result'
    except:
        print 'exception:' + word
        print traceback.format_exc()
        FilmDetail = Object.extend('HJWordStudyCList')
        mFilmDetail = FilmDetail()
        mFilmDetail.set('word', word)
        mFilmDetail.set('category', category)
        mFilmDetail.set('type', type_code)
        mFilmDetail.set('source_url', url)
        mFilmDetail.save()
예제 #14
0
def index():
    college_news = Query(Object.extend('College')).descending(key='news_time'). \
        limit(10).select('news_title', 'news_time').find()

    work_news = Query(Object.extend('Work')).descending(key='news_time'). \
        limit(10).select('news_title', 'news_time').find()

    notice_news = Query(Object.extend('College')).equal_to(key='news_type', value=10).\
        descending(key='news_time').limit(10).select('news_title', 'news_time').find()

    recruit_news = Query(Object.extend('Work')).equal_to(key='news_type', value=16).\
        descending(key='news_time').limit(10).select('news_title', 'news_time').find()

    return render_template('front/index.html', college_news=college_news, work_news=work_news,
                           notice_news=notice_news, recruit_news=recruit_news)
예제 #15
0
def laifudaoxiaohua():
    global sign
    url = 'http://route.showapi.com/107-32'+ sign
    # print url
    req = requests.get(url)
    # print req.text
    result = json.loads(req.text)
    if 0 == result['showapi_res_code']:
        newslist = result['showapi_res_body']['list']
        for data in newslist:
            if is_exit(data['content'].strip()):
                # print 'exit and continue'
                continue
            else:
                # print data['content'].strip()
                # print data['url']
                Reading = Object.extend('Joke')
                mReading = Reading()
                mReading.set('text', data['content'].strip())
                # mReading.set('img', data['picUrl'])
                mReading.set('source_url', data['url'])
                # mReading.set('publish_time', publish_time)
                mReading.set('type', '5')
                mReading.set('category', '101')
                mReading.save()
예제 #16
0
def test_encode():  # type: () -> None
    Foo = Object.extend("Foo")
    obj = Foo()
    assert utils.encode(obj) == {
        "className": "Foo",
        "__type": "Pointer",
        "objectId": None,
    }

    acl = ACL()
    assert utils.encode(acl) == {}
    acl.set_read_access("xxx", True)
    assert utils.encode(acl) == {"xxx": {"read": True}}

    point = GeoPoint()
    assert utils.encode(point) == {
        "__type": "GeoPoint",
        "longitude": 0,
        "latitude": 0,
    }

    assert utils.encode([obj, acl, point]) == [
        {"className": "Foo", "__type": "Pointer", "objectId": None},
        {"xxx": {"read": True}},
        {"__type": "GeoPoint", "longitude": 0, "latitude": 0},
    ]

    assert utils.encode({"a": obj, "b": acl}) == {
        "a": {"className": "Foo", "__type": "Pointer", "objectId": None},
        "b": {"xxx": {"read": True}},
    }
예제 #17
0
def songci(message):
    cmd, text = parse_cmd_text(message.text)
    if text == None or len(text) == 0:
        bot.sendMessage(chat_id=message.chat.id,
                        reply_to_message_id=message.message_id,
                        text='请使用 /songci <词名>')
        return
    bot.sendChatAction(chat_id=message.chat.id, action=telegram.ChatAction.TYPING)
    text = text.replace(' ', '·')
    keyword = urllib2.quote(text)
    response = urllib2.urlopen(songci_api + keyword)
    data = json.loads(response.read())
    Songci = Object.extend('Songci')
    __songci = Songci()
    __songci.set('keyword', keyword)
    __songci.set('data', response.read())
    __songci.save()
    try:
        a_songci =  data['result']['list'][0]
    except TypeError as e:
        bot.sendMessage(chat_id=message.chat.id,
                        reply_to_message_id=message.message_id,
                        text='找不到对应的宋词')
        return
    __text = a_songci['title'] + '\n' + a_songci['author'] + '\n' + a_songci['content']
    block_chars = '⓪①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳❶❷❸❹❺❻❼❽❾❿⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽⑾⑿⒀⒁⒂⒃⒄⒅⒆⒇'
    temp = ''
    for c in __text:
        if not c in block_chars:
            temp += c
    __text = temp.replace('&nbsp;', ' ').replace('<br />', '\n')
    bot.sendMessage(chat_id=message.chat.id, text=__text)
예제 #18
0
def xiaohuadaquan_gif():
    global sign
    for page in range(1,2):
        para = '&page=%s&maxResult=20' % (page)
        url = 'http://route.showapi.com/341-3'+ sign + para
        # print page
        # print url
        req = requests.get(url)
        # print req.text
        result = json.loads(req.text)
        if 0 == result['showapi_res_code']:
            newslist = result['showapi_res_body']['contentlist']
            for data in newslist:
                if is_exit(data['title'].strip()):
                    # print 'exit and continue'
                    continue
                else:
                    publish_time = datetime.strptime(str(data['ct']), "%Y-%m-%d %H:%M:%S.%f")
                    # print data['title'].strip()
                    # print publish_time
                    # print data['img']
                    ratio = getImageRatio(data['img'])
                    if ratio == 5.5555:
                        continue
                    Reading = Object.extend('Joke')
                    mReading = Reading()
                    mReading.set('text', data['title'].strip())
                    mReading.set('img', data['img'])
                    mReading.set('ratio', ratio)
                    # mReading.set('source_url', data['url'])
                    mReading.set('publish_time', publish_time)
                    mReading.set('type', '3')
                    mReading.set('category', '101')
                    mReading.save()
def analyze_tracker_static_info(table_name=APP_STATIC_INFO_TABLE):
    DBTable = Object.extend(table_name)

    print 'begin get_all_trackers'
    application_tracker_dict = get_all_trackers(db_name=INSTALLATION_TABLE)
    print 'end get_all_trackers'
    for app,tracker_list in application_tracker_dict.items():
        dbTable = DBTable()
        print 'begin get_age_and_gender_data_dict'
        age_and_gender_dict = get_age_and_gender_data_dict(table_name=WEIGHTED_STATIC_INFO,tracker_list=tracker_list)
        occupation_dict = get_occupation_data_dict(table_name=WEIGHTED_STATIC_INFO,tracker_list=tracker_list)
        sport_dict = get_sport_data_dict(table_name=WEIGHTED_STATIC_INFO,tracker_list=tracker_list)
        consumption_dict = get_consumption_data_dict(table_name=WEIGHTED_STATIC_INFO,tracker_list=tracker_list)
        field_dict = get_field_data_dict(table_name=WEIGHTED_STATIC_INFO,tracker_list=tracker_list)


        print 'end get_age_and_gender_data_dict'
        dbTable.set('age_and_gender',age_and_gender_dict)
        dbTable.set('occupation',occupation_dict)
        dbTable.set('sport',sport_dict)
        dbTable.set('consumption',consumption_dict)
        dbTable.set('field',field_dict)
        dbTable.set('app',app)
        relation = dbTable.relation('tracker')
        for tracker in tracker_list:
            relation.add(tracker)
        dbTable.save()
예제 #20
0
파일: app.py 프로젝트: aisk/URL-Shortener
def gen_short_url(lurl):
    '''Generates shortened URL from given long url, returns a string as the
    shortened url's key.
    '''
    try:
        if get_long(lurl) is not None: return lurl
    except LeanCloudError as e:
        if e.code == 101:
            pass
        else:
            raise e
    try:
        return get_short(lurl).get('short')
    except LeanCloudError as e:
        if e.code == 101:
            pass
        else:
            raise e
    Shortened = Object.extend('Shortened')
    shortened = Shortened()
    # Hard coded size. Fix later ( or never )
    size = 4
    surl = gen_random_string(size=size)
    shortened.set('long', lurl)
    shortened.set('short', surl)
    shortened.save()
    return surl
예제 #21
0
 def process_item(self, item, spider):
     global itemId
     print '-----------------process_item-----------------------'
     if is_exit(item['title'], item['category']):
         print 'already exit'
         return item
     else:
         itemId += 1
         Composition = Object.extend('Reading')
         mComposition = Composition()
         mComposition.set('item_id', itemId)
         mComposition.set('title', item['title'])
         mComposition.set('img_url', item['img_url'])
         mComposition.set('content', item['content'])
         mComposition.set('type_name', item['type_name'])
         mComposition.set('publish_time', item['publish_time'])
         mComposition.set('type_id', item['type_id'])
         mComposition.set('source_url', item['source_url'])
         mComposition.set('source_name', item['source_name'])
         mComposition.set('category', item['category'])
         mComposition.set('type', item['type'])
         mComposition.set('img_type', 'url')
         mComposition.set('img_urls', item['img_urls'])
         mComposition.set('media_url', item['media_url'])
         mComposition.set('category_2', '')
         mComposition.save()
         print('save item')
         return item
def see_record_num(period_label_dic):
	valid_num=0
	
	valid_period=[] 
	####init
	connect_db()
	#####get all
	UserSensor = Object.extend('UserSensor')
	query = Query(UserSensor)
	#
	for label,periods in period_label_dic.items():
	
		for period in periods:
			stamp_range=generate_stamp(period)
			#
			query.equal_to("deviceId",device).not_equal_to("events",None).\
			equal_to("sensorType",sensor).\
			less_than("timestamp", stamp_range[1]).greater_than("timestamp",stamp_range[0])
			#equal_to("motion",class_type).\
			
			if query.count()>10:
				valid='valid'
				valid_num+=query.count()
				valid_period.append([label,period])
			else:valid='not valid'
			print label,period,'count',query.count(),valid
	print 'total valid record:',valid_num
예제 #23
0
def nowplaying_movies(url, typeId, publish_time):
    global item_id
    contents = ''
    img_url = ''
    type_name = ''
    # browser = webdriver.PhantomJS(executable_path='/root/phantomjs/bin/phantomjs')
    browser = webdriver.PhantomJS(
        executable_path=
        '/Users/luli/Downloads/phantomjs-2.1.1-macosx/bin/phantomjs')
    browser.get(url)
    # print 'current_url:'+browser.current_url.encode('utf-8')
    # time.sleep(3)

    try:
        title = browser.find_element_by_css_selector('h1.ph').text

        if is_exit(title):
            # print('already exit'.encode('utf-8'))
            return
        else:
            soup = BeautifulSoup(browser.page_source, "html5lib")
            wholeLayer = soup.find('div', id='wholeLayer').find_all(
                'td', attrs={"align": "left"})
            contents = ''
            if len(wholeLayer) <= 0:
                return

            for td in wholeLayer:
                if td.text.strip():
                    contents += td.text.strip() + '\n\n'

            type_name = get_type_name(typeId)
            print(title.encode('utf-8'))
            print(contents.encode('utf-8'))
            print('img_url:' + img_url.encode('utf-8'))
            print(typeId.encode('utf-8'))
            print(type_name.encode('utf-8'))
            item_id += 1
            Composition = Object.extend('Reading')
            mComposition = Composition()
            mComposition.set('item_id', item_id)
            mComposition.set('title', title)
            mComposition.set('img_url', img_url)
            mComposition.set('img_type', 'url')
            mComposition.set('content', contents)
            mComposition.set('type_name', type_name)
            mComposition.set('type_id', str(typeId))
            mComposition.set('publish_time', publish_time)
            mComposition.set('source_url', url)
            mComposition.set('source_name', '酷悠双语网')
            mComposition.set('category', 'shuangyu_reading')
            mComposition.set('type', 'text')
            # mComposition.save()
    except:
        print 'except return'.encode('utf-8')
        # time.sleep(5)
        # nowplaying_movies(url,typeId,publish_time)
        return

    browser.quit()
def get_all_behavior_prediction(table_name='UserBehavior',field_name='prediction'):
    DBTable = Object.extend(table_name)
    query = Query(DBTable)
    query.exists(field_name)
    query.less_than('updatedAt',currentTime)
    total_count=query.count()
    print 'TotalCount  %s' %str(total_count)

    query_times=(total_count+query_limit-1)/query_limit

    behavior_prediction_dict={}
    for index in range(query_times):
        print 'querying index: %s' %str(index)
        query = Query(DBTable)
        query.exists(field_name)
        query.less_than('updatedAt',currentTime)
        query.limit(query_limit)
        query.skip(index*query_limit)
        query.descending('updatedAt')
        result_list=query.find()
        for result in result_list:
            record_wrap ={}
            record_wrap['startTime'] = result.get('startTime')
            record_wrap['endTime'] = result.get('endTime')
            record_wrap['prediction'] = result.get(field_name)
            # record_wrap['UserBehavior'] = result   # 这里是为了绑定userBehavior
            behavior_prediction_dict[result]=record_wrap  #这里的键user由于是对象,所以即使是同一个user,他们也是不同的
    return behavior_prediction_dict
예제 #25
0
def getPositionLeanCloud():
    Position = Object.extend('Position')
    position = Query(Position)
    position.select('position')
    query_list = position.find()
    #    print (str(query_list[0]))
    return query_list
예제 #26
0
def saveBalanceLeanCloud(b):
    #    print('balance',b)
    if b:
        Balance = Object.extend('Balance')
        balance = Balance()
        balance.set('balance', b[0])
        balance.save()
예제 #27
0
def get_daocloud_app():
    print('get func begin run..')
    base_url = 'https://openapi.daocloud.io/v1/apps'
    DAOCLOUD_APITOKEN = os.environ.get('DAOCLOUD_APITOKEN')
    token = "token " + DAOCLOUD_APITOKEN
    requests.packages.urllib3.disable_warnings()
    result = requests.get(base_url, headers={"Authorization": token})
    data = json.loads(result.text)
    DaoCloudApp = Object.extend('DaoCloudApp')
    query = leancloud.Query('DaoCloudApp')
    for daoapp in data['app']:
        query.equal_to("appid", daoapp['id'])
        try:
            dao = query.first()
            dao.set('state', daoapp['state'])
            dao.set('last_operated_at', daoapp['last_operated_at'])
            dao.save()
        except leancloud.LeanCloudError as e:
            if e.code == 101:
                dao = DaoCloudApp()
                dao.set('appid', daoapp['id'])
                dao.set('name', daoapp['name'])
                dao.set('state', daoapp['state'])
                dao.set('last_operated_at', daoapp['last_operated_at'])
                dao.save()
            else:
                raise e
    print('get func OK')
예제 #28
0
def laifudaotupianxiaohua():
    global sign
    url = 'http://route.showapi.com/107-33'+ sign
    # print url
    req = requests.get(url)
    # print req.text
    result = json.loads(req.text)
    if 0 == result['showapi_res_code']:
        newslist = result['showapi_res_body']['list']
        for data in newslist:
            if is_exit(data['title'].strip()):
                # print 'exit and continue'
                continue
            else:
                time.sleep(3)
                # print data['title'].strip()
                # print data['thumburl']
                ratio = getImageRatio(data['thumburl'])
                Reading = Object.extend('Joke')
                mReading = Reading()
                mReading.set('text', data['title'].strip())
                mReading.set('img', data['thumburl'])
                mReading.set('ratio', ratio)
                mReading.set('type', '1')
                mReading.set('category', '101')
                if ratio != 5.5555:
                    mReading.save()
예제 #29
0
def saveZhongJiang(user, msg):

    Balance = Object.extend('ZhongJiang')
    balance = Balance()
    balance.set('msg', msg)
    balance.set('user', user)
    balance.save()
예제 #30
0
def sync_data_table_update(startDateTime, endDateTime, formType):
    '''
        同步数据库表信息(每隔一天抓取)、同步昨天 00:00:00 至 今天天 23:59:59 增加以及更新的数据
    '''
    startStr = startDateTime.strftime('%Y-%m-%d')
    rc = checkWriteState(startStr, formType, operateType[0])
    if rc is True:
        Log(content='数据库表 ' + formType + ' ' + startStr + ' 更新数据已同步!!',
            fromTask='system', parameter='', createTime=datetime.now()).save()
        print '数据库表 ' + formType + ' ' + startStr + ' 更新数据已同步!!'
        return
    # 开始同步数据库表的操作
    obj = Object.extend(formType)
    query = Query(obj)
    # 包括新创建的和原来已有的,但昨天之前更新的数据
    if rc is False:
        query.greater_than_or_equal_to('createdAt', startDateTime)
        query.less_than('updatedAt', endDateTime)
    elif rc is None:  # 说明数据没有进行同步操作, 那么抓去全部的数据。
        query.less_than('createdAt', datetime.now())
        # query.less_than('createdAt', datetime(year=2016, month=8, day=15))
    def update_local_data(lst):
        dataList = []
        for item in lst:
            recordMap = covert_obj_to_dict(item)
            dataList.append(recordMap)
        recordsCount = len(dataList)
        if recordsCount > 0:
            __update_local_data(dataList, formType)  # 将更新后的数据写入本地mangodb
        Log(content=formType + ' 表本次一共存储了 ' + str(recordsCount) + ' 条记录!',
            fromTask='system', parameter='', createTime=datetime.now()).save()
        print formType + ' 表本次一共存储了 ' + str(recordsCount) + ' 条记录!'

    recordsCount = map_query(query, 1000, update_local_data)
    finishLoadData(startStr, formType, recordsCount, operateType[0])
예제 #31
0
def get_daily_checkin(start, end):
    global dcCount
    dcCount = 0
    formType = 'DailyCheckIn'
    dateStr = start.strftime('%Y-%m-%d')
    rc = checkWriteState(dateStr, formType)
    if rc:
        print 'already batch inserted'
        return
    print '开始存储 ' + dateStr + ' ' + formType + ' 的数据'
    ckObj = Object.extend('DailyCheckIn')
    ckQuery = Query(ckObj)
    ckQuery.greater_than_or_equal_to('createdAt',start)
    ckQuery.less_than_or_equal_to('createdAt',end)
    def call_back(lst):
        global dcCount
        writeList =  []
        for item in lst:
            ufMap = {}
            ufMap['openId'] = item.get('openId')
            ufMap['checkTime'] = item.get('checkTime')
            ufMap['srcId'] = item.id
            ufMap['createdAt'] = item.created_at
            ufMap['updatedAt'] = item.updated_at
            writeList.append(ufMap)
        dcCount = dcCount + len(writeList)
        if len(writeList) > 0:
            loadDailyCheckIn(writeList)
    map_query(ckQuery, 1000, call_back)
    finishLoadData(dateStr, formType, dcCount)
예제 #32
0
    def get_sport_data_dict(self, app_table=APPLICATION_CLASS, field_name="app", kind=None):
        try:
            field = "sport"
            app = self.get_the_app(kind=kind)
            static_info_table = STATIC_INFO_TABLE
            DbTable = Object.extend(static_info_table)
            query = Query(DbTable)
            query.equal_to(field_name, app)
            query.exists(field)
            result_list = query.find()
            length = len(result_list)
            if length == 0:
                print "error: application not exists in table %s" % (str(static_info_table))
                return {}
            elif length > 1:
                print "error: multi application  exists in table %s" % (str(static_info_table))
                # return 0

            app_static_info = result_list[0]
            data_dict = app_static_info.get(field)
            # return age_and_gender_dict

        except LeanCloudError, e:

            raise e
def get_field_data_dict(table_name='WeightedStaticInfo',tracker_list=None):
    try:
        field_name = 'field'
        tracker_field_name = 'user'
        DBTable  = Object.extend(table_name)

        data_dict ={}
        # new_data_dict = {key:[0 for i in range(4)] for key in dataDict.keys()}
        total_count = len(tracker_list)
        for index, tracker in enumerate(tracker_list):
            query = Query(DBTable)
            query.equal_to(tracker_field_name,tracker)
            query.select(field_name)
            result_list = query.find()
            length = len(result_list)
            # 以后这个地方的判断还需要修改
            if length!=1:
                print 'error: the length of result_list is %s with index: %s with tracker_objectId: %s' %(str(length),str(index),tracker.id)
            if length >=1:
                result = result_list[0]
            else:
                continue
            print 'index: %s  field_name: %s: result: %s  ' %(str(index),str(field_name),str(result.get(field_name)))
            field_key= result.get(field_name).keys()[0]
            if field_key in data_dict.keys():
                data_dict[field_key] += 1
            else:
                data_dict[field_key] = 1
        known_count = sum(data_dict.values())
        data_dict['unknown'] = total_count - known_count
    except LeanCloudError, e:
         raise e
예제 #34
0
def saveErrorLog(user, msg):

    Balance = Object.extend('ErrorLog')
    balance = Balance()
    balance.set('msg', msg)
    balance.set('user', user)
    balance.save()
예제 #35
0
    def create_new_app(self, app_name):
        try:
            if not app_name:
                return 0
            user = self.user.become(self.session_token)
            print "Got the user"
            Application = Object.extend(APPLICATION_CLASS)
            application = Application()
            query = Query(Application)
            query.equal_to("user", user)
            query.equal_to("app_name", app_name)
            if query.find():
                print "Application name exists!"
                return 0
            else:
                print "Application name not exists! "
                application.set("app_name", app_name)
                application.set("user", user)
                application.save()
                app_id = application.id
                app_key = (signer.sign(app_id).split(app_id + "."))[1]
                # app_key = app_id+"this is app_key"

                application.set("app_id", app_id)
                application.set("app_key", app_key)
                application.save()
                return 1
        except LeanCloudError, e:
            print e
            return 0
예제 #36
0
def saveQiangquan(user, msg):

    Balance = Object.extend('Qiangquan')
    balance = Balance()
    balance.set('msg', msg)
    balance.set('user', user)
    balance.save()
예제 #37
0
파일: userer.py 프로젝트: heamon7/bbsUser
    def __init__(self):
        leancloud.init('mctfj249nwy7c1ymu3cps56lof26s17hevwq4jjqeqoloaey', master_key='ao6h5oezem93tumlalxggg039qehcbl3x3u8ofo7crw7atok')

        Users = Object.extend('Users')
        query = Query(Users)
        curTime = datetime.now()
        query.exists('userId')
        query.less_than('createdAt',curTime)
        userCount = query.count()


        print "userCounts: %s" %str(userCount)
        queryLimit = 500
        queryTimes = userCount/queryLimit + 1
        self.urls = []

        for index in range(queryTimes):
            query = Query(Users)
            query.exists('userId')
            query.less_than('createdAt',curTime)
            query.descending('createdAt')
            query.limit(queryLimit)
            query.skip(index*queryLimit)
            query.select('userId')
            userRet = query.find()
            for user in userRet:
                self.urls.append(self.baseUrl + user.get('userId') +".json")
        pass
예제 #38
0
def after_event_save(event):
    print("after event save started")
    institution = event.get('institution')
    print "institution:" + institution
    Conversation = Object.extend('_Conversation')
    query1 = Conversation.query
    query2 = Conversation.query
    query1.equal_to('name', institution)
    query2.equal_to('sys', True)
    query = Query.and_(query1, query2)
    query_list = query.find()
    if len(query_list) == 0:
        raise LeanEngineError('没有找到系统对话')
    else:
        conversation = query_list[0]
        conversation_id = conversation.get('objectId')
        print "conversationId:" + conversation_id
        eventId = event.get('objectId')
        headers = {'Content-Type': 'application/json', \
            'X-LC-Id': APP_ID, \
            'X-LC-Key': MASTER_KEY + ',master'}
        data = {"from_peer": "sys", \
                "message": "{\"_lctype\":-1,\"_lctext\":\"eventCreated\", \
                \"_lcattrs\":{\"snType\":"                                           + str(snEventCreated) + \
                ",\"eventId\": \"" + eventId + "\"}}", \
                 "conv_id": conversation_id}
        requests.post(subscriber_url, data=json.dumps(data), headers=headers)
    print("after event save ended")
예제 #39
0
def get_user_file(start, end):
    global ufCount
    ufCount = 0
    formType = 'UserFile'
    dateStr = start.strftime('%Y-%m-%d')
    rc = checkWriteState(dateStr, formType)
    if rc:
        print 'already batch inserted'
        return
    print '开始存储 ' + dateStr + ' ' + formType + ' 的数据'
    #判断今天是不是已经搞定了
    ufObj = Object.extend('UserFile')
    ufQuery = Query(ufObj)
    ufQuery.greater_than_or_equal_to('createdAt',start)
    ufQuery.less_than_or_equal_to('createdAt',end)

    def uf_call_back(lst):
        global ufCount
        writeList =  []
        for item in lst:
            ufMap = {}
            ufMap['openId'] = item.get('openId')
            ufMap['fileType'] = item.get('fileType')
            ufMap['srcId'] = item.id
            ufMap['createdAt'] = item.created_at
            ufMap['updatedAt'] = item.updated_at
            writeList.append(ufMap)
        ufCount = ufCount + len(writeList)
        if len(writeList) > 0:
            loadUserFile(writeList)
    map_query(ufQuery, 1000, uf_call_back)
    print '本次一共存储了' + str(ufCount) + '条纪录' #一共存储了多少
    finishLoadData(dateStr, formType, ufCount)
예제 #40
0
    def get_event_to_location_data(self, event_name=None, app_table=APPLICATION_CLASS, kind=None):

        # print app_table
        try:

            app = self.get_the_app(kind=kind)
            db_name = "FakeEventActivity"
            DbTable = Object.extend(db_name)
            query = Query(DbTable)
            # 这里只是测试知道是少于1K条的
            query.equal_to("application", app)
            query.exists("location_dict")
            if event_name:
                query.equal_to("event_name", event_name)
            # query.equal_to('application_id',application_id)
            query.descending("createdAt")
            query.limit(1)
            result_list = query.find()
            if result_list:
                # event_name = result_list[0].get('event_name')
                # activity_statistics_dict = {result_list[0].get('event_name'):result_list[0].get('activity_dict')}
                statistics_dict = result_list[0].get("location_dict")
            else:
                statistics_dict = []
        except LeanCloudError, e:
            raise e
예제 #41
0
    def __init__(self):
        leancloud.init(settings.APP_ID, master_key=settings.MASTER_KEY)
        QuestionFollowee = Object.extend('Followee' + self.userDataId)
        questionFollowee = QuestionFollowee()  #
        query = Query(QuestionFollowee)

        query.equal_to('followerLinkId',self.userLinkId)
        curTime = datetime.now()
        query.less_than('createdAt', curTime)
        followeeCount = query.count()
        print "followeeNumbers: %s" % str(questionNum)
        queryLimit = 500
        queryTimes = int(followeeCount +queryLimit -1) / queryLimit
        self.urls = []
        for index in range(queryTimes):
            query = Query(QuestionFollowee)
            query.less_than('createdAt', curTime)
            query.equal_to('followerLinkId',self.userLinkId)
            query.descending('createdAt')
            query.limit(queryLimit)
            query.skip(index * queryLimit)
            query.select('followeeDataId')
            quesRet = query.find()
            for ques in quesRet:
                self.followeeDataIdList.extend(ques.get('followeeDataId'))
예제 #42
0
    def get_tracker_of_app(self, app_id=""):
        try:
            self.tracker_list = []
            # user = self.user.become(self.session_token)
            Application = Object.extend("Application")
            query = Query(Application)
            # query.equal_to('user', user)

            # app_id = 'demo 55f7e36f60b2fe7115171b4b'
            print "@@@@@@@@@@@@@@@@@@" + app_id
            query.equal_to("app_id", app_id)
            app_list = query.find()

            if len(app_list) != 1:
                return []
            the_app = app_list[0]

            # Tracker = Object.extend('BindingInstallation')
            # query = Query(Tracker)
            # query.equal_to('application', the_app)
            # app_list = query.find()
            relation = the_app.relation("tracker")
            print relation,
            return 1
        except LeanCloudError, e:
            print e
            return 0
예제 #43
0
def checkIfConfirmationCodeMatches(**params):
    print "check if confirmation code matches starts"
    if 'email' in params and 'code' in params:
        try:
            email = params['email']
            print "email: " + email
            code = params['code']
            print "code: " + code
            ConfirmationCode = Object.extend('ConfirmationCode')
            twentyMinutesAgo = datetime.datetime.now() - datetime.timedelta(
                minutes=20)
            print(twentyMinutesAgo)
            query1 = ConfirmationCode.query
            query2 = ConfirmationCode.query
            query3 = ConfirmationCode.query
            query1.equal_to('email', email)
            query2.equal_to('code', code)
            query3.greater_than_or_equal_to('updatedAt', twentyMinutesAgo)
            query12 = Query.and_(query1, query2)
            query = Query.and_(query12, query3)
            query_list = query.find()
            print "check if confirmation code matches ends"

            if len(query_list) == 0:
                return False
            else:
                return True
        except Exception as e:
            print e
            print "check if confirmation code matches ends"
            raise LeanEngineError('系统错误:无法验证验证码')
    else:
        print "email and code cannot be empty"
        print "check if confirmation code matches ends"
        raise LeanEngineError('邮箱以及验证码都不能为空')
예제 #44
0
def start_if_stop():
    print('start func begin run..')
    base_url = 'https://openapi.daocloud.io/v1/apps'
    DAOCLOUD_APITOKEN = os.environ.get('DAOCLOUD_APITOKEN')
    token = "token {token}".format(token=DAOCLOUD_APITOKEN)
    requests.packages.urllib3.disable_warnings()
    result = requests.get(base_url, headers={"Authorization": token})
    data = json.loads(result.text)
    DaoCloudApp = Object.extend('DaoCloudApp')
    query = leancloud.Query('DaoCloudApp')
    for daoapp in data['app']:
        try:
            if daoapp['state'] == 'stopped':
                print(daoapp['name'] + ' stopped, start...')
                action_id_text = requests.post(
                    '{0}/{1}/actions/start'.format(base_url, daoapp['id']),
                    headers={"Authorization": token}).text
                action_id = json.loads(action_id_text)
                action_result = 'in_process'
                while action_result == 'in_process':
                    action_result_text = requests.get(
                        '{0}/{1}/actions/{2}'.format(base_url,
                                                     daoapp['id'], action_id['action_id']),
                        headers={"Authorization": token}).text
                    action_result = json.loads(action_result_text)['state']
                    print(action_result)
                    time.sleep(5)
                print(daoapp['name'] + ' ' + action_result)
        except leancloud.LeanCloudError as e:
            raise e
    print('start func OK')
예제 #45
0
def tupianxiaohua():
    global sign
    for page in range(1,2):
        para = '&page=%s&maxResult=20&time=2015-07-10' % (page)
        url = 'http://route.showapi.com/341-2'+ sign + para
        # print page
        # print url
        req = requests.get(url)
        # print req.text
        result = json.loads(req.text)
        if 0 == result['showapi_res_code']:
            newslist = result['showapi_res_body']['contentlist']
            for data in newslist:
                if is_exit(data['title'].strip()):
                    # print 'exit and continue'
                    continue
                else:
                    # print data['title'].strip()
                    # print data['img']
                    ratio = getImageRatio(data['img'])
                    if ratio == 5.5555:
                        continue
                    Reading = Object.extend('Joke')
                    mReading = Reading()
                    mReading.set('text', data['title'].strip())
                    mReading.set('img', data['img'])
                    mReading.set('ratio', ratio)
                    mReading.set('type', '1')
                    mReading.set('category', '101')
                    mReading.save()
def get_tracker_data(table_name=None,tracker_list=None,field_name=None):

    DBTable = Object.extend(table_name)
    tracker_data_dict = {}
    for index,tracker in enumerate(tracker_list):
        #这样处理是因为可能一个user的记录超过了一次可以读取的数量(1K条)
        query = Query(DBTable)
        query.equal_to(field_name,tracker)
        query.less_than('createdAt',currentTime)
        total_count=query.count()
        # print 'TotalCount  %s' %str(total_count)

        query_times=(total_count+query_limit-1)/query_limit
        #如果想在这里按timestamp排序取出每个user的记录是不靠谱的,可能读取时还有插入,而且timestamp介于之间
        for index in range(query_times):
            # print 'querying index: %s' %str(index)
            query = Query(DBTable)
            query.equal_to(field_name,tracker)
            query.less_than('createdAt',currentTime)
            query.ascending('createdAt')
            query.limit(query_limit)
            query.skip(index*query_limit)
            if tracker in userDataDict.keys():
                tracker_data_dict.get(tracker).extend(query.find())
            else :
                tracker_data_dict[tracker]=query.find()
    return tracker_data_dict
def get_occupation_data_dict(table_name='WeightedStaticInfo',tracker_list=None):
    try:
        field_name = 'user'
        DBTable  = Object.extend(table_name)
        gender_type_list =['man','woman']
        age_type_list = ['16down','16to35','35to55','55up']
        dataDict ={gender_type:{age_type:0 for age_type in age_type_list} for gender_type in gender_type_list}
        # new_data_dict = {key:[0 for i in range(4)] for key in dataDict.keys()}
        total_count = len(tracker_list)
        for index, tracker in enumerate(tracker_list):
            query = Query(DBTable)
            query.equal_to(field_name,tracker)
            query.select('age','gender')
            result_list = query.find()
            length = len(result_list)
            if length!=1:
                print 'error: the length of result_list is %s with index: %s with tracker_objectId: %s' %(str(length),str(index),tracker.id)
            if length >=1:
                result = result_list[0]
            else:
                continue

            print 'index: %s  gender: %s  age: %s ' %(str(index),str(result.get('gender')),str(result.get('age')))
            gender = 'man' if result.get('gender') >0 else 'woman'
            age_info_dict= result.get('age')
            dataDict[gender][age_info_dict.keys()[0]] += 1
            # dataDict ={'man' if staticInfo.get('gender') >0 else 'woman':dataDict['man' if staticInfo.get('gender') >0 else 'woman'][staticInfo.get('age').keys()[0]] +=1 for staticInfo in staticInfoList}

            # for index ,age_type in enumerate(age_type_list):
            #     for gender_type in dataDict.keys():
            #         new_data_dict[gender_type][index] = dataDict[gender_type][age_type]
        known_count = sum(dataDict['man'].values())+sum(dataDict['woman'].values())
        dataDict['unknown'] = total_count -known_count
    except LeanCloudError, e:
         raise e
예제 #48
0
def query2():
    location_list = []
    finger_print = Object.extend('fingerprint')
    my_query = finger_print.query
    my_query.select('name', 'info')
    query_list = my_query.find()
    for test in query_list:
        temp_dict = {}
        name = test.get('name')
        fp = test.get('info')
        temp_dict['Tag'] = name
        temp_dict['FingerPrint'] = eval(fp)
        # print(temp_dict['FingerPrint'])
        location_list.append(temp_dict)
        # print(name, fp)
    for location in location_list:
        # if re.search('[7][0-9]{3}', location['Tag']):
        for fp in location['FingerPrint']:
            if fp['mac'] not in mac_list:
                mac_list.append(fp['mac'])
    for location in location_list:
        for mac in mac_list:
            flag = 1
            for fp in location['FingerPrint']:
                if mac == fp['mac']:
                    flag = 0
                    break
            if flag:
                temp_dict = {'mac': mac, 'rss': '-99'}
                location['FingerPrint'].append(temp_dict)
    return location_list
def get_tracker_of_app(app_id):
    query = Query(Object.extend('Application'))
    query.equal_to('app_id', app_id)
    app_list = query.find()
    if not app_list:
        return []
    the_app = app_list[0]

    query = Query(Object.extend('BindingInstallation'))
    query.equal_to('application', the_app)
    query.select('user')
    installation_list = query.find()
    user_set = set()
    for installation in installation_list:
        user_set.add(installation.attributes['user'].id)
    return list(user_set)
예제 #50
0
    def process_item(self, item, spider):
        global itemId
        print '-----------------process_item-----------------------'
        if is_exit(item['title'],item['category']):
            print 'already exit'
            return item
        else:
            content = ''
            contents = item['content']
            for con in contents.splitlines():
                content += con.strip()
                content += '\n\n'

            itemId += 1
            Composition = Object.extend('Reading')
            mComposition = Composition()
            mComposition.set('item_id', itemId)
            mComposition.set('title', item['title'])
            mComposition.set('img_url', item['img_url'])
            mComposition.set('content', content)
            mComposition.set('type_name', item['type_name'])
            mComposition.set('publish_time', item['publish_time'])
            mComposition.set('type_id', item['type_id'])
            mComposition.set('source_url', item['source_url'])
            mComposition.set('source_name', item['source_name'])
            mComposition.set('category', item['category'])
            mComposition.set('type', item['type'])
            mComposition.set('img_type', 'url')
            mComposition.set('media_url', item['media_url'])
            # mComposition.set('category_2', '')
            mComposition.save()
            print('save item')
            return item
예제 #51
0
def parse_detail(url, title, des, cover, likes, author):
    headers = {
        'User-Agent':
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36"
    }
    req = requests.get(url, headers=headers)
    req.encoding = 'utf-8'
    soup = BeautifulSoup(req.text, "html5lib")

    # print req.text
    views = 0
    viewstr = ''
    read_url = ''
    a_tag = soup.select('div.btn.btn-primary a')
    if len(a_tag) > 0:
        read_url = urlparse.urljoin(url, a_tag[0]['href'])

    viewstr = likes
    if u'亿' in viewstr:
        viewstr = viewstr.replace(u'亿', '')
        views = float(viewstr) * 100000000
    elif u'万' in viewstr:
        viewstr = viewstr.replace(u'万', '')
        views = float(viewstr) * 10000
    elif u',' in viewstr:
        viewstr = viewstr.replace(u',', '')
        views = float(viewstr)
    else:
        views = float(viewstr)

    print title
    print author
    print read_url
    print url
    print views
    print likes
    print des
    if is_exit(title, url):
        print 'url is exit'
        return

    Composition = Object.extend('Caricature')
    mComposition = Composition()
    mComposition.set('name', title)
    mComposition.set('author', author)
    mComposition.set('book_img_url', cover)
    mComposition.set('category', "")
    mComposition.set('des', des)
    mComposition.set('read_url', read_url)
    mComposition.set('tag', "")
    mComposition.set('isFree', '')
    mComposition.set('type', "")
    mComposition.set('viewstr', likes)
    mComposition.set('update', "")
    mComposition.set('url', url)
    mComposition.set('source_name', "快看漫画")
    mComposition.set('views', views)
    mComposition.save()
    print('save item')
예제 #52
0
def nowplaying_movies(url,publish_time):
    global item_id
    contents = ''
    img_url = ''
    type_name = ''
    headers = {'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'}
    req = requests.get(url,headers=headers)
    # req.encoding='utf-8'
    soup = BeautifulSoup(req.text,"html5lib")
    [script.extract() for script in soup.findAll('script')]
    [style.extract() for style in soup.findAll('style')]

    title = soup.find('div',class_='list_left_list').find('h1').text

    if is_exit(title,publish_time):
        print('already exit')
        return

    your_string = soup.find('div',class_='contentmain').get_text()
    p=re.compile('\s+ ')
    contents = re.sub(p,'',your_string)
    contents = contents.replace('\n\n\n', '\n')

    type_name = soup.find('li',id='select').find('a').text
    # p = soup.find('div',class_='contentmain').find_all('p')
    # if len(p) > 0:
    #     for i in range(0,len(p)):
    #         contents += p[i].text
    #         contents += '\n'
    # else:
    #     contents = soup.find('div',class_='content-ad').next_sibling
    #     contents += '\n'
    #     contents += soup.find('div',class_='content-ad').next_sibling.next_sibling.next_sibling

    typeId = get_type_id(type_name)

    print(title)
    print(type_name)
    print(typeId)
    print(item_id)
    print(contents)

    item_id += 1
    Composition = Object.extend('Reading')
    mComposition = Composition()
    mComposition.set('item_id', item_id)
    mComposition.set('title', title)
    mComposition.set('img_url', img_url)
    mComposition.set('img_type', 'url')
    mComposition.set('content', contents)
    mComposition.set('type_name', type_name)
    mComposition.set('type_id', typeId)
    mComposition.set('publish_time', publish_time)
    mComposition.set('source_url', url)
    mComposition.set('source_name', '水滴英语作文网')
    mComposition.set('category', 'composition')
    mComposition.set('type', 'text')
    mComposition.save()
    print('save item')
예제 #53
0
 def set(self, key, value, ttl=None):
     try:
         obj = Query(self.class_name).equal_to('key', key).first()
     except LeanCloudError:
         obj = Object.extend(self.class_name)()
     obj.set('key', key)
     obj.set('value', value)
     obj.save()
예제 #54
0
def test_save_date(): # type: () -> None
    DateObject = Object.extend('DateObject')
    now = datetime.now()

    d = DateObject(date=now)
    d.save()
    server_date = Query(DateObject).get(d.id).get('date')
    assert now.isoformat().split('.')[0] == server_date.isoformat().split('.')[0]
예제 #55
0
def saveAccount(user, password):

    Balance = Object.extend('Account')
    balance = Balance()
    balance.set('user', user)
    balance.set('password', password)
    balance.set('flag', '0')
    balance.save()
예제 #56
0
파일: xxoo.py 프로젝트: zzljob/xxoo
def saveImage(url):
    ImageObject = Object.extend('XXXiaoImage')
    img = ImageObject()
    img.set("url", url)
    try:
        img.save()
    except LeanCloudError, e:
        print e