Exemplo n.º 1
0
 def __parse_base(self, item, city, point_type):
     point = Point()
     point.prov = self.uid
     point.type = point_type
     point.name = normalize_text(item('.b-map-side>h5').text())
     point.address, point.place = split_address_place(u'г. %s, %s' % (city, item('.b-map-side>p span:eq(0)').text()))
     coordinates = item('.b-map-side>p span:eq(1)').text()
     if coordinates:
         point.lat, point.lng = map(strip, coordinates.split(','))
     text_html = replace_br(item('.b-map-side-more').html(), ';;;')
     time_items = []
     for sub_item in map(normalize_text, PQ(text_html).text().split(';;;')):
         if not sub_item:
             continue
         if sub_item.startswith(u'Телефон:'):
             point.phones = normalize_phones(sub_item[len(u'Телефон:')].split(','))
             continue
         time_items.append(sub_item)
     point.time = normalize_time(', '.join(time_items))
     point.check_information = CHECK_OFFICIAL
     if point.lat and point.lng:
         point.check_coordinates = CHECK_OFFICIAL
     else:
         warning_not_official_coordinates(point)
     return point
Exemplo n.º 2
0
    def __parse_base(self, item, city_name, point_type):
        point = Point()
        point.prov = self.uid
        point.type = point_type

        point.phones = [normalize_phone(item('.content_table table tbody tr:eq(0) td:eq(0) .office_phone').remove().text())]
        name_address_html = replace_br(item('.content_table table tbody tr:eq(0) td:eq(0)').remove().html(), ',')
        name, address = PQ(name_address_html).text().split(',', 1)
        point.name = normalize_text(name)
        point.address, point.place = self.__get_address(city_name, address)
        point.check_information = CHECK_OFFICIAL

        script_text = item('.ya_map script:eq(1)').text()
        for line in map(strip, script_text.splitlines()):
            if line.startswith('BX_GMapAddPlacemark('):
                lat_token = "'LAT':'"
                lat_start_index = line.find(lat_token) + len(lat_token)
                lat_end_index = line.find("'", lat_start_index)
                point.lat = line[lat_start_index:lat_end_index]
                lng_token = "'LON':'"
                lng_start_index = line.find(lng_token) + len(lng_token)
                lng_end_index = line.find("'", lng_start_index)
                point.lng = line[lng_start_index:lng_end_index]
                point.check_coordinates = CHECK_OFFICIAL
                break
        else:
            warning_not_official_coordinates(point)
        return point
Exemplo n.º 3
0
 def __parse_phones(self, item):
     if len(item('td')) > 1:
         more_info_html = replace_br(item('td:last').html(), ';;;')
         more_info_items = map(strip, PQ(more_info_html).text().split(';;;'))
         if more_info_items and more_info_items[0].startswith('('):
             return normalize_phones(more_info_items[0].split(','))
     return []
Exemplo n.º 4
0
 def get_offices(self):
     points = []
     items_tree = ET.fromstring(get_url(self.__offices_xml_url))
     for item in items_tree.iter('item'):
         point = self.__parse_office(item)
         if point:
             points.append(point)
     page = PQ(get_url(self.__regional_offices_page_url))
     point = None
     for item in map(PQ, page('#content_internal span:eq(0)').children()):
         if item[0].tag not in self.__regional_offices_tags:
             continue
         if item[0].tag == 'h2':
             point = Point()
             point.prov = self.uid
             point.type = TYPE_OFFICE
             point.name = trim_spaces_and_commas(normalize_text(item.text()))
             point.check_information = CHECK_OFFICIAL
             continue
         if not point:
             continue
         item_html = replace_br(item.html(), ';;;')
         sub_items = PQ(item_html).text().split(';;;')
         point.address, point.place = split_address_place(sub_items[0])
         for sub_item in map(normalize_text, sub_items[1:]):
             if sub_item.startswith(u'т.ф.:'):
                 point.phone = normalize_phones(sub_item[len(u'т.ф.:'):].split(','))
         warning_not_official_coordinates(point)
         points.append(point)
         point = None
     return points
Exemplo n.º 5
0
 def __parse_office(self, item):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_OFFICE
     point.name = normalize_text(item('h1').text())
     point.address, point.place =  split_address_place(item('tr:eq(2) td:eq(1)').text())
     phones = []
     phone_html = replace_br(item('tr:eq(5) td:eq(1)').html(), ';;;')
     if phone_html:
         phones += map(strip, PQ(phone_html).text().split(';;;'))
     phone_html = replace_br(item('tr:eq(6) td:eq(1)').html(), ';;;')
     if phone_html:
         phones += map(strip, PQ(phone_html).text().split(';;;'))
     point.phones = normalize_phones(filter(lambda phone: phone.startswith((u'+', u'тел')), phones))
     point.time = normalize_time(item('tr:eq(8) td:eq(1)').text())
     point.check_information = CHECK_OFFICIAL
     warning_not_official_coordinates(point)
     return point
Exemplo n.º 6
0
 def __parse_office(self, item, city):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_OFFICE
     point.name = normalize_text(item('th .pointShowMaps span:eq(0)').text())
     address = item('th .pointShowMaps span:eq(1)').text()
     point.address, point.place = split_address_place(u'г. %s, %s' % (city, address))
     time_html = replace_br(item('td:eq(0)').html(), ', ')
     point.time = normalize_time(PQ(time_html).text())
     phones_html = replace_br(item('td:eq(1)').html(), ', ')
     point.phones = normalize_phones(PQ(phones_html).text().split(','))
     point.lat = normalize_text(item('th .item_coords .coord1').text())
     point.lng = normalize_text(item('th .item_coords .coord2').text())
     point.check_information = CHECK_OFFICIAL
     if point.lat and point.lng:
         point.check_coordinates = CHECK_OFFICIAL
     else:
         warning_not_official_coordinates(point)
     return point
Exemplo n.º 7
0
 def __parse_office(self, item):
     point, more = self.__parse_base(item)
     point.type = TYPE_OFFICE
     point.check_information = CHECK_OFFICIAL
     for section in map(PQ, more(".content .section")):
         section_type = normalize_text(section(".name").text())
         if section_type not in self.__office_sections_types:
             continue
         section_value = section(".text")
         type = self.__office_sections_types[section_type]
         if type == "time":
             time_html = replace_br(section_value.html(), ",")
             time_text = ", ".join([item.text() for item in map(PQ, PQ(time_html)("td"))])
             point.time = normalize_time(time_text)
         elif type == "phone":
             phones_html = replace_br(section_value.html(), ",")
             phones_text = normalize_text(PQ(phones_html).text())
             point.phones = filter_empty(map(normalize_phone, phones_text.split(",")))
     warning_not_official_coordinates(point)
     return point
Exemplo n.º 8
0
 def __parse_terminal(self, item, map_points):
     point = self.__parse_base_atm_terminals(item, map_points, TYPE_TERMINAL, self.__terminal_names)
     if not point:
         return None
     item_html = replace_br(item('.addres').html(), ',')
     for text in map(normalize_text, PQ(item_html).text().split(',')[1:]):
         if text.startswith(u'Взнос наличных:'):
             point.deposit = normalize_text(text.replace(u'Взнос наличных:', '')).lower() == u'да'
             continue
         if text.startswith(u'Время работы:'):
             point.time = normalize_time(text.replace(u'Время работы:', ''))
             continue
     return point
Exemplo n.º 9
0
 def __parse_atm(self, item, map_points):
     point = self.__parse_base_atm_terminals(item, map_points, TYPE_ATM, self.__atms_names)
     if not point:
         return None
     item_html = replace_br(item('.addres').html(), ',')
     for text in map(normalize_text, PQ(item_html).text().split(',')[1:]):
         if text.startswith(u'Валюта:'):
             point.currency = map(normalize_text, text.replace(u'Валюта:', '').replace(u'Бел. Рубли', 'BYR').split(','))
             continue
         if text.startswith(u'Время работы:'):
             point.time = normalize_time(text.replace(u'Время работы:', ''))
             continue
     return point
Exemplo n.º 10
0
 def __parse_time(self, time):
     if not time.html():
         return None
     time_html = replace_br(time.html(), ',')
     time_items = []
     for time_item in map(strip, PQ(time_html).text().split(',')):
         lower_time_item = time_item.lower()
         if lower_time_item.endswith(':'):
             if lower_time_item.startswith(self.__skip_time_tokens):
                 continue
             if not lower_time_item.startswith(self.__normal_time_tokens):
                 break
         time_items.append(time_item)
     return normalize_time(', '.join(time_items))
Exemplo n.º 11
0
 def __parse_office(self, item, city_name):
     point = self.__parse_base(item, city_name, TYPE_OFFICE)
     for sub_item in map(PQ, item('.content_table table tbody tr')):
         if normalize_text(sub_item('td:eq(0)').text()) == u'Кассы':
             time_items = [normalize_text(PQ(replace_br(sub_item('td:eq(1)').html(), ',')).text())]
             break_time = normalize_text(sub_item('td:eq(2)').text())
             if break_time:
                 time_items.append(u'перерыв: ' + break_time)
             day_off = normalize_text(sub_item('td:eq(3)').text())
             if day_off:
                 time_items.append(u'выходной: ' + day_off)
             point.time = normalize_time(', '.join(time_items))
             break
     else:
         return None
     return point
Exemplo n.º 12
0
 def __parse_office_exchange(self, item, city, coordinates, point_type, point_keywords):
     point = Point()
     point.prov = self.uid
     point.type = point_type
     point.name = normalize_text(item('th:eq(0)').text())
     if not point.name.startswith(point_keywords):
         return None
     address_html = replace_br(item('td:eq(0)').html(), ';;;')
     address_items = PQ(address_html).text().split(';;;', 1)
     point.address = normalize_address(u'%s, %s' % (city, address_items[0]))
     if len(address_items) > 1:
         point.place = normalize_text(address_items[1])
     item('td:eq(1) ul, td:eq(1) li').remove()
     point.time = normalize_time(item('td:eq(1)').text())
     point.phones = normalize_phones(map(lambda phone_item: PQ(phone_item).text(), item('td:eq(2) p') or item('td:eq(2)')))
     point.check_information = CHECK_OFFICIAL
     point.lat, point.lng = self.__get_point_coordinate(point, coordinates)
     if point.lat and point.lng:
         point.check_coordinates = CHECK_OFFICIAL
     else:
         warning_not_official_coordinates(point)
     return point
Exemplo n.º 13
0
 def __parse_atm(self, item, city):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_ATM
     address = item('th .pointShowMaps span').remove().text()
     place = normalize_text(item('th .pointShowMaps').text())
     point.address, point.place = split_address_place(u'г. %s, %s' % (city, address))
     point.place = place
     currency = item('td:eq(0)').text()
     for from_token, to_token in self.__currency_replaces:
         currency = currency.replace(from_token, to_token)
     point.currency = map(strip, currency.split(','))
     time_html = replace_br(item('td:eq(1)').html(), ', ')
     point.time = normalize_time(PQ(time_html).text())
     point.lat = normalize_text(item('th .item_coords .coord1').text())
     point.lng = normalize_text(item('th .item_coords .coord2').text())
     point.check_information = CHECK_OFFICIAL
     if point.lat and point.lng:
         point.check_coordinates = CHECK_OFFICIAL
     else:
         warning_not_official_coordinates(point)
     return point
Exemplo n.º 14
0
 def __parse_base_atm_terminal(self, item, city_name, point_type):
     point = self.__parse_base(item, city_name, point_type)
     point.place = item('td:eq(1)').text()
     point.currency = filter_empty(map(normalize_text, PQ(replace_br(item('td:eq(2)').html(), ',')).text().split(',')))
     point.time = self.__parse_time(item('td:eq(3)'))
     return point