def __parse_base(self, item, city, point_type): point = Point() point.prov = self.uid point.type = point_type point.name = normalize_text(item('.b-map-side>h5').text()) point.address, point.place = split_address_place(u'г. %s, %s' % (city, item('.b-map-side>p span:eq(0)').text())) coordinates = item('.b-map-side>p span:eq(1)').text() if coordinates: point.lat, point.lng = map(strip, coordinates.split(',')) text_html = replace_br(item('.b-map-side-more').html(), ';;;') time_items = [] for sub_item in map(normalize_text, PQ(text_html).text().split(';;;')): if not sub_item: continue if sub_item.startswith(u'Телефон:'): point.phones = normalize_phones(sub_item[len(u'Телефон:')].split(',')) continue time_items.append(sub_item) point.time = normalize_time(', '.join(time_items)) point.check_information = CHECK_OFFICIAL if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_base(self, item, city_name, point_type): point = Point() point.prov = self.uid point.type = point_type point.phones = [normalize_phone(item('.content_table table tbody tr:eq(0) td:eq(0) .office_phone').remove().text())] name_address_html = replace_br(item('.content_table table tbody tr:eq(0) td:eq(0)').remove().html(), ',') name, address = PQ(name_address_html).text().split(',', 1) point.name = normalize_text(name) point.address, point.place = self.__get_address(city_name, address) point.check_information = CHECK_OFFICIAL script_text = item('.ya_map script:eq(1)').text() for line in map(strip, script_text.splitlines()): if line.startswith('BX_GMapAddPlacemark('): lat_token = "'LAT':'" lat_start_index = line.find(lat_token) + len(lat_token) lat_end_index = line.find("'", lat_start_index) point.lat = line[lat_start_index:lat_end_index] lng_token = "'LON':'" lng_start_index = line.find(lng_token) + len(lng_token) lng_end_index = line.find("'", lng_start_index) point.lng = line[lng_start_index:lng_end_index] point.check_coordinates = CHECK_OFFICIAL break else: warning_not_official_coordinates(point) return point
def __parse_phones(self, item): if len(item('td')) > 1: more_info_html = replace_br(item('td:last').html(), ';;;') more_info_items = map(strip, PQ(more_info_html).text().split(';;;')) if more_info_items and more_info_items[0].startswith('('): return normalize_phones(more_info_items[0].split(',')) return []
def get_offices(self): points = [] items_tree = ET.fromstring(get_url(self.__offices_xml_url)) for item in items_tree.iter('item'): point = self.__parse_office(item) if point: points.append(point) page = PQ(get_url(self.__regional_offices_page_url)) point = None for item in map(PQ, page('#content_internal span:eq(0)').children()): if item[0].tag not in self.__regional_offices_tags: continue if item[0].tag == 'h2': point = Point() point.prov = self.uid point.type = TYPE_OFFICE point.name = trim_spaces_and_commas(normalize_text(item.text())) point.check_information = CHECK_OFFICIAL continue if not point: continue item_html = replace_br(item.html(), ';;;') sub_items = PQ(item_html).text().split(';;;') point.address, point.place = split_address_place(sub_items[0]) for sub_item in map(normalize_text, sub_items[1:]): if sub_item.startswith(u'т.ф.:'): point.phone = normalize_phones(sub_item[len(u'т.ф.:'):].split(',')) warning_not_official_coordinates(point) points.append(point) point = None return points
def __parse_office(self, item): point = Point() point.prov = self.uid point.type = TYPE_OFFICE point.name = normalize_text(item('h1').text()) point.address, point.place = split_address_place(item('tr:eq(2) td:eq(1)').text()) phones = [] phone_html = replace_br(item('tr:eq(5) td:eq(1)').html(), ';;;') if phone_html: phones += map(strip, PQ(phone_html).text().split(';;;')) phone_html = replace_br(item('tr:eq(6) td:eq(1)').html(), ';;;') if phone_html: phones += map(strip, PQ(phone_html).text().split(';;;')) point.phones = normalize_phones(filter(lambda phone: phone.startswith((u'+', u'тел')), phones)) point.time = normalize_time(item('tr:eq(8) td:eq(1)').text()) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_office(self, item, city): point = Point() point.prov = self.uid point.type = TYPE_OFFICE point.name = normalize_text(item('th .pointShowMaps span:eq(0)').text()) address = item('th .pointShowMaps span:eq(1)').text() point.address, point.place = split_address_place(u'г. %s, %s' % (city, address)) time_html = replace_br(item('td:eq(0)').html(), ', ') point.time = normalize_time(PQ(time_html).text()) phones_html = replace_br(item('td:eq(1)').html(), ', ') point.phones = normalize_phones(PQ(phones_html).text().split(',')) point.lat = normalize_text(item('th .item_coords .coord1').text()) point.lng = normalize_text(item('th .item_coords .coord2').text()) point.check_information = CHECK_OFFICIAL if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_office(self, item): point, more = self.__parse_base(item) point.type = TYPE_OFFICE point.check_information = CHECK_OFFICIAL for section in map(PQ, more(".content .section")): section_type = normalize_text(section(".name").text()) if section_type not in self.__office_sections_types: continue section_value = section(".text") type = self.__office_sections_types[section_type] if type == "time": time_html = replace_br(section_value.html(), ",") time_text = ", ".join([item.text() for item in map(PQ, PQ(time_html)("td"))]) point.time = normalize_time(time_text) elif type == "phone": phones_html = replace_br(section_value.html(), ",") phones_text = normalize_text(PQ(phones_html).text()) point.phones = filter_empty(map(normalize_phone, phones_text.split(","))) warning_not_official_coordinates(point) return point
def __parse_terminal(self, item, map_points): point = self.__parse_base_atm_terminals(item, map_points, TYPE_TERMINAL, self.__terminal_names) if not point: return None item_html = replace_br(item('.addres').html(), ',') for text in map(normalize_text, PQ(item_html).text().split(',')[1:]): if text.startswith(u'Взнос наличных:'): point.deposit = normalize_text(text.replace(u'Взнос наличных:', '')).lower() == u'да' continue if text.startswith(u'Время работы:'): point.time = normalize_time(text.replace(u'Время работы:', '')) continue return point
def __parse_atm(self, item, map_points): point = self.__parse_base_atm_terminals(item, map_points, TYPE_ATM, self.__atms_names) if not point: return None item_html = replace_br(item('.addres').html(), ',') for text in map(normalize_text, PQ(item_html).text().split(',')[1:]): if text.startswith(u'Валюта:'): point.currency = map(normalize_text, text.replace(u'Валюта:', '').replace(u'Бел. Рубли', 'BYR').split(',')) continue if text.startswith(u'Время работы:'): point.time = normalize_time(text.replace(u'Время работы:', '')) continue return point
def __parse_time(self, time): if not time.html(): return None time_html = replace_br(time.html(), ',') time_items = [] for time_item in map(strip, PQ(time_html).text().split(',')): lower_time_item = time_item.lower() if lower_time_item.endswith(':'): if lower_time_item.startswith(self.__skip_time_tokens): continue if not lower_time_item.startswith(self.__normal_time_tokens): break time_items.append(time_item) return normalize_time(', '.join(time_items))
def __parse_office(self, item, city_name): point = self.__parse_base(item, city_name, TYPE_OFFICE) for sub_item in map(PQ, item('.content_table table tbody tr')): if normalize_text(sub_item('td:eq(0)').text()) == u'Кассы': time_items = [normalize_text(PQ(replace_br(sub_item('td:eq(1)').html(), ',')).text())] break_time = normalize_text(sub_item('td:eq(2)').text()) if break_time: time_items.append(u'перерыв: ' + break_time) day_off = normalize_text(sub_item('td:eq(3)').text()) if day_off: time_items.append(u'выходной: ' + day_off) point.time = normalize_time(', '.join(time_items)) break else: return None return point
def __parse_office_exchange(self, item, city, coordinates, point_type, point_keywords): point = Point() point.prov = self.uid point.type = point_type point.name = normalize_text(item('th:eq(0)').text()) if not point.name.startswith(point_keywords): return None address_html = replace_br(item('td:eq(0)').html(), ';;;') address_items = PQ(address_html).text().split(';;;', 1) point.address = normalize_address(u'%s, %s' % (city, address_items[0])) if len(address_items) > 1: point.place = normalize_text(address_items[1]) item('td:eq(1) ul, td:eq(1) li').remove() point.time = normalize_time(item('td:eq(1)').text()) point.phones = normalize_phones(map(lambda phone_item: PQ(phone_item).text(), item('td:eq(2) p') or item('td:eq(2)'))) point.check_information = CHECK_OFFICIAL point.lat, point.lng = self.__get_point_coordinate(point, coordinates) if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_atm(self, item, city): point = Point() point.prov = self.uid point.type = TYPE_ATM address = item('th .pointShowMaps span').remove().text() place = normalize_text(item('th .pointShowMaps').text()) point.address, point.place = split_address_place(u'г. %s, %s' % (city, address)) point.place = place currency = item('td:eq(0)').text() for from_token, to_token in self.__currency_replaces: currency = currency.replace(from_token, to_token) point.currency = map(strip, currency.split(',')) time_html = replace_br(item('td:eq(1)').html(), ', ') point.time = normalize_time(PQ(time_html).text()) point.lat = normalize_text(item('th .item_coords .coord1').text()) point.lng = normalize_text(item('th .item_coords .coord2').text()) point.check_information = CHECK_OFFICIAL if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_base_atm_terminal(self, item, city_name, point_type): point = self.__parse_base(item, city_name, point_type) point.place = item('td:eq(1)').text() point.currency = filter_empty(map(normalize_text, PQ(replace_br(item('td:eq(2)').html(), ',')).text().split(','))) point.time = self.__parse_time(item('td:eq(3)')) return point