def __parse_exchange(self, item, city): point = Point() point.prov = self.uid point.type = TYPE_EXCHANGE point.name = normalize_text(item('td:eq(0)').text()) point.address, point.place = split_address_place(u'г. %s, %s' % (city, item('td:eq(1)').text())) if len(item('td')) == 4: point.time = normalize_time(item('td:eq(2)').text()) else: point.time = normalize_time(item('td:eq(2)').text().split(u'Операции:')[0]) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_base_atm_terminal(self, row, point_type, coordinates, deposit=False): point = Point() point.prov = self.uid point.type = point_type point.name = normalize_text(u'№' + str(int(row[1]))) city = row[2] if u'р-н' not in row[2]: city = u'г. %s' % city point.address = normalize_address(u'%s, %s' % (city, row[3])) point.place = normalize_text(row[4]) if u'только безнал.платежи' in row[5]: point.currency = [] if deposit: point.deposit = False else: point.currency = map(strip, row[5].split(',')) if deposit: point.deposit = True point.time = normalize_time(row[6]) point.check_information = CHECK_OFFICIAL point.lat, point.lng = self.__get_point_coordinate(point.address, coordinates) if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_base_office_exchange(self, item, map_points, point_type, start_names): point = Point() point.prov = self.uid point.type = point_type point.name = normalize_text(item('.name').text()) if not point.name.startswith(start_names): return None point.address, point.place = split_address_place(item('.addres strong').text()) sub_item = item('.item_block tr:last') point.phones = normalize_phones(sub_item('td:eq(0)').text().split(',')) mon_thu = u'пн-чт: ' + sub_item('td:eq(2)').text() fri = u'пт: ' + sub_item('td:eq(3)').text() sat = u'сб: ' + sub_item('td:eq(4)').text() sun = u'вс: ' + sub_item('td:eq(5)').text() point.time = normalize_time(', '.join([mon_thu, fri, sat, sun])) point.check_information = CHECK_OFFICIAL for lng, lat, name, address, place in map_points: if (point.name in name if point.name and name else True) and\ (point.address and address and point.address in address) and\ (point.place in place if point.place and place else True): point.lat = lat point.lng = lng point.check_coordinates = CHECK_OFFICIAL break else: warning_not_official_coordinates(point) return point
def __parse_terminal(self, item): point = Point() point.prov = self.uid point.type = TYPE_TERMINAL city = normalize_text(item('td:eq(0)').text()) address = normalize_text(item('td:eq(2)').text()) point.address, point.place = split_address_place(u'г. %s, %s' % (city.title(), address)) point.place = normalize_text(item('td:eq(1)').text()) point.time = normalize_time(item('td:eq(3)').text()) point.check_information = CHECK_OFFICIAL for lat, lng, type_id, description in self.__get_coordinates(): if u'Минск' not in point.address or type_id != '2': continue for token in description.split(): if token not in point.address: break else: point.lat = lat point.lng = lng point.check_coordinates = CHECK_OFFICIAL break else: warning_not_official_coordinates(point) return point
def __parse_time(self, item): start_times = map(lambda sub_item: normalize_text(PQ(sub_item).text()), item('td:eq(2) p') or item('td:eq(2)')) end_times = map(lambda sub_item: normalize_text(PQ(sub_item).text()), item('td:eq(3) p') or item('td:eq(3)')) break_times = map(lambda sub_item: normalize_text(PQ(sub_item).text()), item('td:eq(4) p') or item('td:eq(4)')) weekends = normalize_text(item('td:eq(5)').text()) time = u'' if len(start_times) == 1: sub_start_day, sub_start_time = self.__split_day_time(start_times[0]) sub_break_day, sub_break_time = self.__split_day_time(break_times[0]) if break_times[0] != u'без обеда' else (None, None) for end_time in end_times: sub_end_day, sub_end_time = self.__split_day_time(end_time) if sub_end_day: time += u'%s: %s-%s, ' % (sub_end_day, sub_start_time, sub_end_time) else: time += u'%s-%s, ' % (sub_start_time, sub_end_time) if sub_break_time: time += u'обед: %s, ' % sub_break_time else: for start_time in start_times: sub_start_day, sub_start_time = self.__split_day_time(start_time) for end_time in end_times: sub_end_day, sub_end_time = self.__split_day_time(end_time) if not sub_end_day or sub_start_day == sub_end_day: time += u'%s: %s-%s, ' % (sub_start_day, sub_start_time, sub_end_time) for break_time in break_times: sub_break_day, sub_break_time = self.__split_day_time(break_time) if break_time != u'без обеда' else (None, None) if sub_break_time and sub_start_day == sub_break_day: time += u'обед: %s, ' % sub_break_time if weekends != u'без выходных': time += u'%s: выходной' % weekends return normalize_time(time)
def __get_offices(self, url, city_name=''): points = [] page = PQ(get_url(url).decode('utf8')) time = None for item in map(PQ, page('#oo__content_value table tr:gt(0)')): if item('td').attr('colspan') == '3': continue point = Point() point.prov = self.uid point.type = TYPE_OFFICE point.name = normalize_text(item('td:eq(0)').text()) point.address = normalize_address(city_name + item('td:eq(1) p:eq(0)').text()) place = item('td:eq(1) p:eq(2)').text() if not place: place = item('td:eq(1) p:eq(1)').text() if place: point.place = normalize_text(place) new_time = item('td:eq(2)').text() if new_time: time = new_time point.time = normalize_time(time) point.check_information = CHECK_OFFICIAL if point.address in self.__addresses: point.lat, point.lng = self.__addresses[point.address] point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) points.append(point) return points
def __parse_base(self, item, city, point_type): point = Point() point.prov = self.uid point.type = point_type point.name = normalize_text(item('.b-map-side>h5').text()) point.address, point.place = split_address_place(u'г. %s, %s' % (city, item('.b-map-side>p span:eq(0)').text())) coordinates = item('.b-map-side>p span:eq(1)').text() if coordinates: point.lat, point.lng = map(strip, coordinates.split(',')) text_html = replace_br(item('.b-map-side-more').html(), ';;;') time_items = [] for sub_item in map(normalize_text, PQ(text_html).text().split(';;;')): if not sub_item: continue if sub_item.startswith(u'Телефон:'): point.phones = normalize_phones(sub_item[len(u'Телефон:')].split(',')) continue time_items.append(sub_item) point.time = normalize_time(', '.join(time_items)) point.check_information = CHECK_OFFICIAL if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_base_atm_terminal_point(self, item, city_name, point_type): point = self.__parse_base(item, city_name, point_type) point.time = normalize_time(item('.content_table table tbody tr td:eq(0)').text()) currencies = item('.content_table table tbody tr td:eq(1)').text() for from_text, to_text in self.__currency_replaces: currencies = currencies.replace(from_text, to_text) point.currency = map(strip, currencies.split(',')) return point
def __parse_base_office_exchange(self, item): point = Point() point.prov = self.uid point.name = normalize_text(item('td:eq(1)').text()) point.address, point.place = split_address_place(item('td:eq(2)').text()) point.time = normalize_time(item('td:eq(3)').text()) point.phones = normalize_phones(item('td:eq(4)').text().split(',')) point.check_information = CHECK_OFFICIAL return point
def __parse_atm(self, item): point = Point() point.prov = self.uid point.type = TYPE_ATM point.address, point.place = split_address_place(item('td:eq(1)').text()) point.time = normalize_time(item('td:eq(2)').text()) point.currency = map(strip, item('td:eq(3)').text().split(',')) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_terminal(self, item): point = Point() point.prov = self.uid point.type = TYPE_TERMINAL point.address, point.place = split_address_place(item('td:eq(1)').text()) point.time = normalize_time(item('td:eq(2)').text()) point.deposit = u'Пополнение карточки наличными' in item('td:eq(3)').text() point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_terminal(self, item): point = Point() point.prov = self.uid point.type = TYPE_TERMINAL city = u'г. %s' % normalize_text(item('td:eq(0)').text()).title() point.address = normalize_address(u'%s, %s' % (city, item('td:eq(1)').text())) point.place = normalize_text(item('td:eq(2)').text()) point.time = normalize_time(item('td:eq(3)').text()) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_atm(self, item): point = Point() point.prov = self.uid point.type = TYPE_ATM city = u'г. %s' % normalize_text(item('td:eq(0)').text()).title() point.address = normalize_address(u'%s, %s' % (city, item('td:eq(1)').text())) point.place = normalize_text(item('td:eq(2)').text()) point.currency = map(strip, item('td:eq(3)').text().split(',')) point.time = normalize_time(item('td:eq(4)').text()) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_terminal(self, item): point = Point() point.prov = self.uid point.type = TYPE_TERMINAL point.name = normalize_text(item('td:eq(0)').text()) point.address, point.place = split_address_place(item('td:eq(1)').text()) point.place = point.name point.time = normalize_time(item('td:eq(2)').text()) point.deposit = normalize_text(item('td:eq(3)').text()).lower() == u'есть' point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_atm(self, item, map_points): point = self.__parse_base_atm_terminals(item, map_points, TYPE_ATM, self.__atms_names) if not point: return None item_html = replace_br(item('.addres').html(), ',') for text in map(normalize_text, PQ(item_html).text().split(',')[1:]): if text.startswith(u'Валюта:'): point.currency = map(normalize_text, text.replace(u'Валюта:', '').replace(u'Бел. Рубли', 'BYR').split(',')) continue if text.startswith(u'Время работы:'): point.time = normalize_time(text.replace(u'Время работы:', '')) continue return point
def __parse_terminal(self, item, map_points): point = self.__parse_base_atm_terminals(item, map_points, TYPE_TERMINAL, self.__terminal_names) if not point: return None item_html = replace_br(item('.addres').html(), ',') for text in map(normalize_text, PQ(item_html).text().split(',')[1:]): if text.startswith(u'Взнос наличных:'): point.deposit = normalize_text(text.replace(u'Взнос наличных:', '')).lower() == u'да' continue if text.startswith(u'Время работы:'): point.time = normalize_time(text.replace(u'Время работы:', '')) continue return point
def __parse_time(self, time): if not time.html(): return None time_html = replace_br(time.html(), ',') time_items = [] for time_item in map(strip, PQ(time_html).text().split(',')): lower_time_item = time_item.lower() if lower_time_item.endswith(':'): if lower_time_item.startswith(self.__skip_time_tokens): continue if not lower_time_item.startswith(self.__normal_time_tokens): break time_items.append(time_item) return normalize_time(', '.join(time_items))
def __parse_terminal(self, item): point = Point() point.prov = self.uid point.type = TYPE_TERMINAL point.address, point.place = split_address_place(item('td:eq(2)').text()) point.place = normalize_text(item('td:eq(1)').text()) point.currency = map(strip, item('td:eq(4)').text().split(',')) if point.currency: point.deposit = True else: point.deposit = False point.time = normalize_time(item('td:eq(3)').text()) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_atm(self, item, city, coordinates): point = Point() point.prov = self.uid point.type = TYPE_ATM point.address = normalize_address(u'%s, %s' % (city, item('td:eq(2)').text())) point.place = normalize_text(item('td:eq(1)').text()) point.currency = map(strip, item('td:eq(4)').text().replace('EURO', 'EUR').split(',')) point.time = normalize_time(item('td:eq(3)').text()) point.check_information = CHECK_OFFICIAL point.lat, point.lng = self.__get_point_coordinate(point, coordinates) if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_office(self, item, city_name): point = self.__parse_base(item, city_name, TYPE_OFFICE) for sub_item in map(PQ, item('.content_table table tbody tr')): if normalize_text(sub_item('td:eq(0)').text()) == u'Кассы': time_items = [normalize_text(PQ(replace_br(sub_item('td:eq(1)').html(), ',')).text())] break_time = normalize_text(sub_item('td:eq(2)').text()) if break_time: time_items.append(u'перерыв: ' + break_time) day_off = normalize_text(sub_item('td:eq(3)').text()) if day_off: time_items.append(u'выходной: ' + day_off) point.time = normalize_time(', '.join(time_items)) break else: return None return point
def __parse_office(self, item): point = Point() point.prov = self.uid point.type = TYPE_OFFICE point.name = normalize_text(item('h1').text()) point.address, point.place = split_address_place(item('tr:eq(2) td:eq(1)').text()) phones = [] phone_html = replace_br(item('tr:eq(5) td:eq(1)').html(), ';;;') if phone_html: phones += map(strip, PQ(phone_html).text().split(';;;')) phone_html = replace_br(item('tr:eq(6) td:eq(1)').html(), ';;;') if phone_html: phones += map(strip, PQ(phone_html).text().split(';;;')) point.phones = normalize_phones(filter(lambda phone: phone.startswith((u'+', u'тел')), phones)) point.time = normalize_time(item('tr:eq(8) td:eq(1)').text()) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def get_atms(self): points = [] page = PQ(get_url(self.__parse_list_atm_url).decode('utf8')) for item in map(PQ, page('#oo__content_value table tr:gt(0)')): point = Point() point.prov = self.uid point.type = TYPE_ATM point.address = normalize_address(item('td:eq(0) p:eq(0)').text()) point.place = normalize_text(item('td:eq(1)').text()) point.time = normalize_time(item('td:eq(2)').text()) point.currency = map(self.__get_currency, item('td:eq(3) p')) point.check_information = CHECK_OFFICIAL if point.address in self.__addresses: point.lat, point.lng = self.__addresses[point.address] point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) points.append(point) return points
def __parse_office(self, item, city): point = Point() point.prov = self.uid point.type = TYPE_OFFICE point.name = normalize_text(item('th .pointShowMaps span:eq(0)').text()) address = item('th .pointShowMaps span:eq(1)').text() point.address, point.place = split_address_place(u'г. %s, %s' % (city, address)) time_html = replace_br(item('td:eq(0)').html(), ', ') point.time = normalize_time(PQ(time_html).text()) phones_html = replace_br(item('td:eq(1)').html(), ', ') point.phones = normalize_phones(PQ(phones_html).text().split(',')) point.lat = normalize_text(item('th .item_coords .coord1').text()) point.lng = normalize_text(item('th .item_coords .coord2').text()) point.check_information = CHECK_OFFICIAL if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def get_exchanges(self): points = [] page = PQ(get_url(self.__parse_list_exchange_url).decode('utf8')) for item in map(PQ, page('#oo__content_value table tr:gt(0)')): point = Point() point.prov = self.uid point.type = TYPE_EXCHANGE add_city_literal = (u'Минск', u'Витебск') address = normalize_text(item('td:eq(0)').text()) point.address = normalize_address((u'г. ' + address) if address.startswith(add_city_literal) else address) point.place = normalize_text(item('td:eq(1)').text()) point.time = normalize_time(item('td:eq(2)').text()) point.check_information = CHECK_OFFICIAL if point.address in self.__addresses: point.lat, point.lng = self.__addresses[point.address] point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) points.append(point) return points
def __parse_office(self, item): point, more = self.__parse_base(item) point.type = TYPE_OFFICE point.check_information = CHECK_OFFICIAL for section in map(PQ, more(".content .section")): section_type = normalize_text(section(".name").text()) if section_type not in self.__office_sections_types: continue section_value = section(".text") type = self.__office_sections_types[section_type] if type == "time": time_html = replace_br(section_value.html(), ",") time_text = ", ".join([item.text() for item in map(PQ, PQ(time_html)("td"))]) point.time = normalize_time(time_text) elif type == "phone": phones_html = replace_br(section_value.html(), ",") phones_text = normalize_text(PQ(phones_html).text()) point.phones = filter_empty(map(normalize_phone, phones_text.split(","))) warning_not_official_coordinates(point) return point
def __parse_exchange(self, item): point = Point() point.prov = self.uid point.type = TYPE_EXCHANGE point.name = normalize_text(item.find('name').text) city = item.find('region').text if item.find('region') else u'Минск' address = item.find('address').text point.address = normalize_address(u'г. %s, %s' % (city.title(), address)) point.place = normalize_text(item.find('location').text) point.lat = item.find('lattitude').text point.lng = item.find('longitude').text point.time = normalize_time(item.find('time').text) if item.find('phones').text: point.phones = normalize_phones(item.find('phones').text.split(',')) point.check_information = CHECK_OFFICIAL if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_base_offices_exchanges(self, item, point_type, keywords_names): point = Point() point.prov = self.uid point.type = point_type point.name = normalize_text(item('.first').text()) if not point.name.startswith(keywords_names): return None city = item('.field-field-city').text() if city: city = u'г. ' + city else: city = item('.field-field-index').text() address = item('.field-field-adress').text() point.address, point.place = split_address_place(u'%s, %s' % (city, address)) phone = item('.field-field-phone').text() if phone: point.phones = [normalize_phone(phone[len(u'тел.:')])] point.time = normalize_time(item('.field-field-work-time .field-item').text()) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_base_office_exchange(self, item, point_type): point = Point() point.prov = self.uid point.type = point_type point.name = normalize_text(item('h2').text()) point.address = normalize_address(item('.itemFilialIn>p:eq(0)').text()[len(u'Почтовый адрес:') + 1:]) is_phone = False phones_items = self.__phone_splitter.split(item('.itemFilialIn>p:eq(1)').text() or '') for sub_item in phones_items: sub_item = normalize_text(sub_item).lower() if sub_item == u'телефон': is_phone = True continue if sub_item == u'факс': is_phone = False continue if is_phone: point.phones.append(normalize_phone(sub_item)) point.time = normalize_time(', '.join(map(lambda sub_item: PQ(sub_item).text(), item('.workTime p')))) point.check_information = CHECK_OFFICIAL warning_not_official_coordinates(point) return point
def __parse_atm(self, item, city): point = Point() point.prov = self.uid point.type = TYPE_ATM address = item('th .pointShowMaps span').remove().text() place = normalize_text(item('th .pointShowMaps').text()) point.address, point.place = split_address_place(u'г. %s, %s' % (city, address)) point.place = place currency = item('td:eq(0)').text() for from_token, to_token in self.__currency_replaces: currency = currency.replace(from_token, to_token) point.currency = map(strip, currency.split(',')) time_html = replace_br(item('td:eq(1)').html(), ', ') point.time = normalize_time(PQ(time_html).text()) point.lat = normalize_text(item('th .item_coords .coord1').text()) point.lng = normalize_text(item('th .item_coords .coord2').text()) point.check_information = CHECK_OFFICIAL if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point
def __parse_office_exchange(self, item, city, coordinates, point_type, point_keywords): point = Point() point.prov = self.uid point.type = point_type point.name = normalize_text(item('th:eq(0)').text()) if not point.name.startswith(point_keywords): return None address_html = replace_br(item('td:eq(0)').html(), ';;;') address_items = PQ(address_html).text().split(';;;', 1) point.address = normalize_address(u'%s, %s' % (city, address_items[0])) if len(address_items) > 1: point.place = normalize_text(address_items[1]) item('td:eq(1) ul, td:eq(1) li').remove() point.time = normalize_time(item('td:eq(1)').text()) point.phones = normalize_phones(map(lambda phone_item: PQ(phone_item).text(), item('td:eq(2) p') or item('td:eq(2)'))) point.check_information = CHECK_OFFICIAL point.lat, point.lng = self.__get_point_coordinate(point, coordinates) if point.lat and point.lng: point.check_coordinates = CHECK_OFFICIAL else: warning_not_official_coordinates(point) return point