def __get_offices(self, url, city_name=''):
points = []
page = PQ(get_url(url).decode('utf8'))
time = None
for item in map(PQ, page('#oo__content_value table tr:gt(0)')):
if item('td').attr('colspan') == '3':
continue
point = Point()
point.prov = self.uid
point.type = TYPE_OFFICE
point.name = normalize_text(item('td:eq(0)').text())
point.address = normalize_address(city_name + item('td:eq(1) p:eq(0)').text())
place = item('td:eq(1) p:eq(2)').text()
if not place:
place = item('td:eq(1) p:eq(1)').text()
if place:
point.place = normalize_text(place)
new_time = item('td:eq(2)').text()
if new_time:
time = new_time
point.time = normalize_time(time)
point.check_information = CHECK_OFFICIAL
if point.address in self.__addresses:
point.lat, point.lng = self.__addresses[point.address]
point.check_coordinates = CHECK_OFFICIAL
else:
warning_not_official_coordinates(point)
points.append(point)
return points
def __parse_base_atm_terminal(self, row, point_type, coordinates, deposit=False):
point = Point()
point.prov = self.uid
point.type = point_type
point.name = normalize_text(u'№' + str(int(row[1])))
city = row[2]
if u'р-н' not in row[2]:
city = u'г. %s' % city
point.address = normalize_address(u'%s, %s' % (city, row[3]))
point.place = normalize_text(row[4])
if u'только безнал.платежи' in row[5]:
point.currency = []
if deposit:
point.deposit = False
else:
point.currency = map(strip, row[5].split(','))
if deposit:
point.deposit = True
point.time = normalize_time(row[6])
point.check_information = CHECK_OFFICIAL
point.lat, point.lng = self.__get_point_coordinate(point.address, coordinates)
if point.lat and point.lng:
point.check_coordinates = CHECK_OFFICIAL
else:
warning_not_official_coordinates(point)
return point
def __parse_terminal(self, item):
point = Point()
point.prov = self.uid
point.type = TYPE_TERMINAL
city = u'г. %s' % normalize_text(item('td:eq(0)').text()).title()
point.address = normalize_address(u'%s, %s' % (city, item('td:eq(1)').text()))
point.place = normalize_text(item('td:eq(2)').text())
point.time = normalize_time(item('td:eq(3)').text())
point.check_information = CHECK_OFFICIAL
warning_not_official_coordinates(point)
return point
def __parse_atm(self, item):
point = Point()
point.prov = self.uid
point.type = TYPE_ATM
point.address = normalize_address(u'г. %s' % item('td:eq(0)').text())
point.place = normalize_text(item('td:eq(1)').text())
point.time = normalize_time(item('td:eq(2)').text())
point.currency = map(strip, item('td:eq(3)').text().split(','))
point.check_information = CHECK_OFFICIAL
warning_not_official_coordinates(point)
return point
def __get_coordinates(self):
if not self.__coordinates:
tree = ET.fromstring(get_url(self.__coordinates_xml_url))
for marker in tree.iter('marker'):
lat = marker.attrib['lat']
lng = marker.attrib['lng']
type_id = marker.attrib['type']
description = normalize_address(marker.text)
for from_token, to_token in self.__description_replaces:
description = description.replace(from_token, to_token)
self.__coordinates.append((lat, lng, type_id, description.strip()))
return self.__coordinates
def __get_coordinates(self):
if not self.__coordinates:
tree = ET.fromstring(get_url(self.__markers_url))
for marker in tree.iter('marker'):
lat = normalize_text(marker.attrib['lat'])
lng = normalize_text(marker.attrib['lng'])
address = normalize_address(marker.attrib['address'])
for from_token, to_token in self.__address_replaces:
address = address.replace(from_token, to_token)
place = normalize_text(marker.attrib['place'])
self.__coordinates.append((lat, lng, address, place))
return self.__coordinates
def __get_coordinates(self):
if not self.__coordinates:
page = PQ(get_url('http://tb.by/private/nal/obmenvalut/'))
script_text = page('script:eq(10)').text()
atm_token = 'newArray[2] ='
exchange_token = 'newArray[22] ='
atm_start = script_text.find(atm_token)
atm_end = script_text.find(';', atm_start + 1)
for item in json.loads(script_text[atm_start + len(atm_token):atm_end]):
lat = item[0].split(',')[0].strip()
lng = item[0].split(',')[1].strip()
address = normalize_address(item[2])
place = normalize_text(item[1])
self.__coordinates.append((lat, lng, address, place))
exchange_start = script_text.find(exchange_token, atm_end + 1)
exchange_end = script_text.find(';', exchange_start + 1)
for item in json.loads(script_text[exchange_start + len(exchange_token):exchange_end]):
lat = item[0].split(',')[0].strip()
lng = item[0].split(',')[1].strip()
address = normalize_address(item[2])
place = normalize_text(item[1])
self.__coordinates.append((lat, lng, address, place))
return self.__coordinates
def __parse_atm(self, item, city, coordinates):
point = Point()
point.prov = self.uid
point.type = TYPE_ATM
point.address = normalize_address(u'%s, %s' % (city, item('td:eq(2)').text()))
point.place = normalize_text(item('td:eq(1)').text())
point.currency = map(strip, item('td:eq(4)').text().replace('EURO', 'EUR').split(','))
point.time = normalize_time(item('td:eq(3)').text())
point.check_information = CHECK_OFFICIAL
point.lat, point.lng = self.__get_point_coordinate(point, coordinates)
if point.lat and point.lng:
point.check_coordinates = CHECK_OFFICIAL
else:
warning_not_official_coordinates(point)
return point
def get_atms(self):
points = []
page = PQ(get_url(self.__parse_list_atm_url).decode('utf8'))
for item in map(PQ, page('#oo__content_value table tr:gt(0)')):
point = Point()
point.prov = self.uid
point.type = TYPE_ATM
point.address = normalize_address(item('td:eq(0) p:eq(0)').text())
point.place = normalize_text(item('td:eq(1)').text())
point.time = normalize_time(item('td:eq(2)').text())
point.currency = map(self.__get_currency, item('td:eq(3) p'))
point.check_information = CHECK_OFFICIAL
if point.address in self.__addresses:
point.lat, point.lng = self.__addresses[point.address]
point.check_coordinates = CHECK_OFFICIAL
else:
warning_not_official_coordinates(point)
points.append(point)
return points
def get_exchanges(self):
points = []
page = PQ(get_url(self.__parse_list_exchange_url).decode('utf8'))
for item in map(PQ, page('#oo__content_value table tr:gt(0)')):
point = Point()
point.prov = self.uid
point.type = TYPE_EXCHANGE
add_city_literal = (u'Минск', u'Витебск')
address = normalize_text(item('td:eq(0)').text())
point.address = normalize_address((u'г. ' + address) if address.startswith(add_city_literal) else address)
point.place = normalize_text(item('td:eq(1)').text())
point.time = normalize_time(item('td:eq(2)').text())
point.check_information = CHECK_OFFICIAL
if point.address in self.__addresses:
point.lat, point.lng = self.__addresses[point.address]
point.check_coordinates = CHECK_OFFICIAL
else:
warning_not_official_coordinates(point)
points.append(point)
return points
def __parse_base_office_exchange(self, item, point_type):
point = Point()
point.prov = self.uid
point.type = point_type
point.name = normalize_text(item('h2').text())
point.address = normalize_address(item('.itemFilialIn>p:eq(0)').text()[len(u'Почтовый адрес:') + 1:])
is_phone = False
phones_items = self.__phone_splitter.split(item('.itemFilialIn>p:eq(1)').text() or '')
for sub_item in phones_items:
sub_item = normalize_text(sub_item).lower()
if sub_item == u'телефон':
is_phone = True
continue
if sub_item == u'факс':
is_phone = False
continue
if is_phone:
point.phones.append(normalize_phone(sub_item))
point.time = normalize_time(', '.join(map(lambda sub_item: PQ(sub_item).text(), item('.workTime p'))))
point.check_information = CHECK_OFFICIAL
warning_not_official_coordinates(point)
return point
def __parse_office_exchange(self, item, city, coordinates, point_type, point_keywords):
point = Point()
point.prov = self.uid
point.type = point_type
point.name = normalize_text(item('th:eq(0)').text())
if not point.name.startswith(point_keywords):
return None
address_html = replace_br(item('td:eq(0)').html(), ';;;')
address_items = PQ(address_html).text().split(';;;', 1)
point.address = normalize_address(u'%s, %s' % (city, address_items[0]))
if len(address_items) > 1:
point.place = normalize_text(address_items[1])
item('td:eq(1) ul, td:eq(1) li').remove()
point.time = normalize_time(item('td:eq(1)').text())
point.phones = normalize_phones(map(lambda phone_item: PQ(phone_item).text(), item('td:eq(2) p') or item('td:eq(2)')))
point.check_information = CHECK_OFFICIAL
point.lat, point.lng = self.__get_point_coordinate(point, coordinates)
if point.lat and point.lng:
point.check_coordinates = CHECK_OFFICIAL
else:
warning_not_official_coordinates(point)
return point
def __parse_exchange(self, item):
point = Point()
point.prov = self.uid
point.type = TYPE_EXCHANGE
point.name = normalize_text(item.find('name').text)
city = item.find('region').text if item.find('region') else u'Минск'
address = item.find('address').text
point.address = normalize_address(u'г. %s, %s' % (city.title(), address))
point.place = normalize_text(item.find('location').text)
point.lat = item.find('lattitude').text
point.lng = item.find('longitude').text
point.time = normalize_time(item.find('time').text)
if item.find('phones').text:
point.phones = normalize_phones(item.find('phones').text.split(','))
point.check_information = CHECK_OFFICIAL
if point.lat and point.lng:
point.check_coordinates = CHECK_OFFICIAL
else:
warning_not_official_coordinates(point)
return point
def __parse_atm(self, item, coordinates):
point = Point()
point.prov = self.uid
point.type = TYPE_ATM
bank = item.find('bank').text
if bank != u'ЗАО БелСвиссБанк':
return None
city = item.find('region').text
address = item.find('address').text
point.address = normalize_address(u'г. %s, %s' % (city.title(), address))
point.place = normalize_text(item.find('location').text)
point.time = normalize_time(item.find('time').text)
point.currency = map(strip, item.find('currency').text.split(','))
point.check_information = CHECK_OFFICIAL
terminal_id = item.find('terminal_id').text
if terminal_id in coordinates:
point.lat, point.lng = coordinates[terminal_id]
if point.lat and point.lng:
point.check_coordinates = CHECK_OFFICIAL
else:
warning_not_official_coordinates(point)
return point