Esempio n. 1
0
        try:
            if '三人' in room.room_type or '3' in room.bed_type:
                room.occupancy = 3
            elif '四人' in room.room_type or '4' in room.bed_type:
                room.occupancy = 4
            else:
                room.occupancy = 2
        except:
            room.occupancy = 2

        room.hotel_name = hotel_name
        room.city = city_name_zh
        room.check_in = check_in
        room.check_out = check_out
        room.source = 'elong'
        room.source_hotelid = hotel_id
        room.currency = 'CNY'
        room.real_source = 'elong'

        room_tuple = (room.hotel_name,room.city,room.source,room.source_hotelid,room.source_roomid, \
            room.real_source,room.room_type,room.occupancy,room.bed_type,room.size,room.floor, \
            room.check_in,room.check_out,room.price,room.tax,room.currency,room.is_extrabed, \
            room.is_extrabed_free,room.has_breakfast,room.is_breakfast_free, \
            room.is_cancel_free,room.room_desc)
        room_list.append(room_tuple)

    return room_list


def get_post_data(hotel_id, check_in, check_out):
    post_data = {
Esempio n. 2
0
                        room.room_type = 'NULL'
            
                if each_room[0] != u'nbsp;':
                    room.price = each_room[0]
                room.has_breakfast = each_room[1]
                room.room_desc = each_room[3]

                if '免费WiFi' in room.room_desc:
                    room.is_wifi_free = 'Yes'
                
                if '免费取消' in room.room_desc:
                    room.is_cancel_free = 'Yes'

                room.currency = 'CNY'
                room.source = 'youzhan'
                room.source_hotelid = hotel_id
                room.check_in = from_date
                room.check_out = to_date

                room_tuple = (room.hotel_name,room.city,room.source,room.source_hotelid,\
                    room.source_roomid,room.real_source,room.room_type,room.occupancy,\
                    room.bed_type,room.size,room.floor,room.check_in,room.check_out,room.price,\
                    room.tax,room.currency,room.is_extrabed,room.is_extrabed_free,room.has_breakfast,\
                    room.is_breakfast_free,room.is_cancel_free,room.room_desc)
                room_list.append(room_tuple)

    else:
        result['error'] = DATA_NONE
        return result

    result['para'] = room_list
Esempio n. 3
0
def youzhan_task_parser(taskcontent):
    all_info = []
    room_list = []
    taskcontent = taskcontent.encode('utf-8').strip()
    hotel_id = taskcontent.split('&')[0]
    star = taskcontent.split('&')[2]
    ipathid = taskcontent.split('&')[1]
    city = taskcontent.split('&')[3]
    country = taskcontent.split('&')[4]
    #room_type = taskcontent.split('&')[3]
    from_date_temp = taskcontent.split('&')[5]
    from_date = from_date_temp[:4] + '-' + from_date_temp[4:6] + '-' \
                + from_date_temp[6:]
    to_date_temp = datetime.datetime(int(from_date_temp[:4]), int(from_date_temp[4:6]), \
                                     int(from_date_temp[6:]))
    to_date = str(to_date_temp + datetime.timedelta(days=1))[:10]

    #获取代理

    p = get_proxy()

    #if p == "":
    #logger.error("get proxy failed")
    #return None

    hotel = Hotel()
    room = Room()

    rating_url = get_rating_url(hotel_id)
    rating_page = crawl_single_page(rating_url, proxy=p)

    grade_str = grade_parser(rating_page)

    if grade_str != '':
        hotel.grade = grade_str[:-1]
    else:
        pass
        #logger.error('Error: No grade_str found!')

    map_url = get_map_url(hotel_id)
    map_page = crawl_single_page(map_url, proxy=p)
    #print map_page
    map_info_list = staticmap_parser(map_page)
    if map_info_list != []:
        hotel.hotel_name = map_info_list[1]
        if is_alphabet(hotel.hotel_name.decode('utf-8')) == True:
            hotel.hotel_name_en = hotel.hotel_name
        else:
            hotel.hotel_name_en = 'NULL'
        hotel.map_info = map_info_list[0]
    else:
        logger.error(
            'youzhanHotel: Map info do not have hotel name and map_info')
        return []

    info_url = get_info_url(hotel_id, from_date, to_date)
    info_page = crawl_single_page(info_url, proxy=p)
    if info_page == '':
        #invalid_proxy(p)
        return []
    info_list = info_parser(info_page)

    if info_list != []:
        hotel.country = country
        hotel.city = city
        hotel.address = info_list[1]
        hotel_desc_temp = info_list[3].replace('<br/>',
                                               '').replace(''', '')
        if hotel_desc_temp != '':
            hotel.description = hotel_desc_temp
        else:
            hotel.description = 'NULL'
        hotel.service = info_list[4]

        if '停车场' in hotel.service:
            hotel.has_parking = 'Yes'
        if '无线网络' in hotel.service or 'wifi' in hotel.service:
            hotel.has_wifi = 'Yes'
    else:
        return []

    hotel.source = 'youzhan'
    hotel.source_id = hotel_id
    hotel.star = star

    price_url = get_price_url(hotel_id, ipathid, from_date, to_date)
    price_page = crawl_single_page(price_url, proxy=p)
    price_list = price_parser(price_page, hotel_id)
    #print '********'
    #print price_list
    if price_list != []:
        for each_room in price_list:
            if len(each_room) > 3:
                room.city = city
                room.occupancy = 2
                room.hotel_name = hotel.hotel_name
                #print '******'
                #print each_room
                room.room_desc = each_room[3]
                room.real_source = each_room[2]

                num = each_room[3].find('-')
                if num > 0:
                    if len(each_room[3][:num]) < 20:
                        room.room_type = each_room[3][:num]
                    else:
                        room.room_type = 'NULL'
                else:
                    if len(each_room[3]) < 20:
                        room.room_type = each_room[3]
                    else:
                        room.room_type = 'NULL'

                if each_room[0] != u'nbsp;':
                    room.price = each_room[0]
                room.has_breakfast = each_room[1]
                room.room_desc = each_room[3]

                if '免费WiFi' in room.room_desc:
                    hotel.is_wifi_free = 'Yes'

                if '免费取消' in room.room_desc:
                    hotel.is_cancel_free = 'Yes'

                room.currency = 'CNY'
                room.source = 'youzhan'
                room.source_hotelid = hotel_id
                room.check_in = from_date
                room.check_out = to_date

                room_tuple = (room.hotel_name,room.city,room.source,room.source_hotelid,\
                    room.source_roomid,room.real_source,room.room_type,room.occupancy,\
                    room.bed_type,room.size,room.floor,room.check_in,room.check_out,room.price,\
                    room.tax,room.currency,room.is_extrabed,room.is_extrabed_free,room.has_breakfast,\
                    room.is_breakfast_free,room.is_cancel_free,room.room_desc)
                room_list.append(room_tuple)

    hotel_tuple = (hotel.hotel_name, hotel.hotel_name_en,hotel.source,hotel.source_id,hotel.brand_name,\
        hotel.map_info,hotel.address,hotel.city,hotel.country,hotel.postal_code, \
        hotel.star,hotel.grade,hotel.has_wifi,hotel.is_wifi_free,hotel.has_parking,\
        hotel.is_parking_free,hotel.service,hotel.img_items,hotel.description)
    hotel_list = []
    hotel_list.append(hotel_tuple)
    all_info.append(hotel_list)
    all_info.append(room_list)

    return all_info
Esempio n. 4
0
def youzhan_task_parser(taskcontent):
    all_info = []
    room_list = []
    taskcontent = taskcontent.encode('utf-8').strip()
    hotel_id = taskcontent.split('&')[0]
    star = taskcontent.split('&')[2]
    ipathid = taskcontent.split('&')[1]
    city = taskcontent.split('&')[3]
    country = taskcontent.split('&')[4]
    #room_type = taskcontent.split('&')[3]
    from_date_temp = taskcontent.split('&')[5]
    from_date = from_date_temp[:4] + '-' + from_date_temp[4:6] + '-' \
                + from_date_temp[6:]
    to_date_temp = datetime.datetime(int(from_date_temp[:4]), int(from_date_temp[4:6]), \
                                     int(from_date_temp[6:]))
    to_date = str(to_date_temp + datetime.timedelta(days = 1))[:10]

    #获取代理
    
    p = get_proxy()

    #if p == "":
        #logger.error("get proxy failed")
        #return None
    
    hotel = Hotel()
    room = Room()

    rating_url = get_rating_url(hotel_id)
    rating_page = crawl_single_page(rating_url, proxy=p)
    
    grade_str = grade_parser(rating_page)
    
    if grade_str != '':
        hotel.grade = grade_str[:-1]
    else:
        pass
        #logger.error('Error: No grade_str found!')

    map_url = get_map_url(hotel_id)
    map_page = crawl_single_page(map_url, proxy=p)
    #print map_page
    map_info_list = staticmap_parser(map_page)
    if map_info_list != []:
        hotel.hotel_name = map_info_list[1]
        if is_alphabet(hotel.hotel_name.decode('utf-8')) == True:
            hotel.hotel_name_en = hotel.hotel_name
        else:
            hotel.hotel_name_en = 'NULL'
        hotel.map_info = map_info_list[0]
    else:
        logger.error('youzhanHotel: Map info do not have hotel name and map_info')
        return []

    info_url = get_info_url(hotel_id,from_date,to_date)
    info_page = crawl_single_page(info_url,proxy=p)
    if info_page == '':
        #invalid_proxy(p)
        return []
    info_list = info_parser(info_page)

    if info_list != []:
        hotel.country = country
        hotel.city = city
        hotel.address = info_list[1]
        hotel_desc_temp = info_list[3].replace('&lt;br/&gt;','').replace('&#039;','')
        if hotel_desc_temp != '':
            hotel.description = hotel_desc_temp
        else:
            hotel.description = 'NULL'
        hotel.service = info_list[4]

        if '停车场' in hotel.service:
            hotel.has_parking = 'Yes'
        if '无线网络' in hotel.service or 'wifi' in hotel.service:
            hotel.has_wifi = 'Yes'
    else:
        return []

    hotel.source = 'youzhan'
    hotel.source_id = hotel_id
    hotel.star = star

    price_url = get_price_url(hotel_id,ipathid,from_date,to_date)
    price_page = crawl_single_page(price_url,proxy=p)
    price_list = price_parser(price_page,hotel_id)
    #print '********'
    #print price_list
    if price_list != []:
        for each_room in price_list:
            if len(each_room) > 3:
                room.city = city
                room.occupancy = 2
                room.hotel_name = hotel.hotel_name
                #print '******'
                #print each_room
                room.room_desc = each_room[3]
                room.real_source = each_room[2]
                

                num = each_room[3].find('-')
                if num > 0:
                    if len(each_room[3][:num]) < 20:
                        room.room_type = each_room[3][:num]
                    else:
                        room.room_type = 'NULL'
                else:
                    if len(each_room[3]) < 20:
                        room.room_type = each_room[3]
                    else:
                        room.room_type = 'NULL'
            
                if each_room[0] != u'nbsp;':
                    room.price = each_room[0]
                room.has_breakfast = each_room[1]
                room.room_desc = each_room[3]

                if '免费WiFi' in room.room_desc:
                    hotel.is_wifi_free = 'Yes'
                
                if '免费取消' in room.room_desc:
                    hotel.is_cancel_free = 'Yes'

                room.currency = 'CNY'
                room.source = 'youzhan'
                room.source_hotelid = hotel_id
                room.check_in = from_date
                room.check_out = to_date

                room_tuple = (room.hotel_name,room.city,room.source,room.source_hotelid,\
                    room.source_roomid,room.real_source,room.room_type,room.occupancy,\
                    room.bed_type,room.size,room.floor,room.check_in,room.check_out,room.price,\
                    room.tax,room.currency,room.is_extrabed,room.is_extrabed_free,room.has_breakfast,\
                    room.is_breakfast_free,room.is_cancel_free,room.room_desc)
                room_list.append(room_tuple)

    hotel_tuple = (hotel.hotel_name, hotel.hotel_name_en,hotel.source,hotel.source_id,hotel.brand_name,\
        hotel.map_info,hotel.address,hotel.city,hotel.country,hotel.postal_code, \
        hotel.star,hotel.grade,hotel.has_wifi,hotel.is_wifi_free,hotel.has_parking,\
        hotel.is_parking_free,hotel.service,hotel.img_items,hotel.description)
    hotel_list = []
    hotel_list.append(hotel_tuple)
    all_info.append(hotel_list)
    all_info.append(room_list)

    return all_info
Esempio n. 5
0
    try:
        each_hotel_content_list = rateplans_pat.findall(content)
        if len(each_hotel_content_list) == 0:
            return all_price
        _INFO('ctripHotel::parseRoom',
              ['Parse price failed because of no hotel found'])
    except Exception, e:
        _ERROR('ctripHotel::parseRoom',
               ['Parse price failed because of no hotel found',
                str(e)])

    for each_hotel_content in each_hotel_content_list:
        room = Room()

        try:
            room.source_hotelid = pattern_search(hotelcode_pat,
                                                 each_hotel_content)
            if room.source_hotelid == 'NULL':
                _INFO('ctripHotel::parseRoom', ['Cannot parse this hotel id'])
        except Exception, e:
            _ERROR(
                'ctripHotel::parseRoom',
                ['Cannot parse this hotel', str(e)])
        #print room.source_hotelid

        try:
            each_room_content_list = rateplan_pat.findall(each_hotel_content)
            if len(each_room_content_list) == 0:
                _ERROR('ctripHotel::parseRoom', ['Parse this room failed'])
                return room_list
        except Exception, e:
            _ERROR('ctripHotel::parseRoom', ['Parse this room failed', e])