def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
if soup != None:
text = json.loads(soup.get_text())
for poi_data in text:
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['street'])
if 'BENU Gyógyszertár' not in poi_data['title']:
name = poi_data['title'].strip()
branch = None
else:
name = 'Benu gyógyszertár'
branch = poi_data['title'].strip()
code = 'hubenupha'
website = poi_data['description'].strip(
) if poi_data['description'] is not None else None
website = website[19:]
nonstop = None
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
city = clean_city(poi_data['city'])
postcode = poi_data['postal_code'].strip()
lat, lon = check_hu_boundary(poi_data['lat'], poi_data['lng'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, postcode)
original = poi_data['street']
ref = None
if 'phone' in poi_data and poi_data['phone'] != '':
phone = clean_phone(poi_data['phone'])
else:
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
xml = save_downloaded_xml(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
root = etree.fromstring(xml)
for e in root.iter('place'):
name = 'MOL Bubi'
code = 'hububibir'
housenumber = None
conscriptionnumber = None
street = None
city = 'Budapest'
branch = e.attrib['name'].split(
'-')[1].strip() if e.attrib['name'] is not None else None
ref = e.attrib['name'].split(
'-')[0].strip() if e.attrib['name'] is not None else None
capacity = e.attrib['bike_racks'].strip(
) if e.attrib['bike_racks'] is not None else None
website = None
nonstop = True
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
lat, lon = check_hu_boundary(e.attrib['lat'].replace(',', '.'),
e.attrib['lng'].replace(',', '.'))
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(self.prefer_osm_postcode,
self.session, lat, lon,
None)
original = None
ref = None
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original, street,
housenumber, conscriptionnumber, ref, phone, email, geom,
nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o, mo_c, th_c,
we_c, tu_c, fr_c, sa_c, su_c
])
print(insert_data)
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
if self.link:
with open(self.link, 'r') as f:
insert_data = []
text = json.load(f)
for poi_data in text['results']:
first_element = next(iter(poi_data))
if self.name == 'CIB bank':
name = 'CIB bank'
code = 'hucibbank'
else:
name = 'CIB'
code = 'hucibatm'
postcode, city, street, housenumber, conscriptionnumber = extract_all_address(
poi_data[first_element]['address'])
branch = None
website = None
nonstop = None
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
lat, lon = check_hu_boundary(
poi_data[first_element]['latitude'],
poi_data[first_element]['longitude'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon,
postcode)
original = poi_data[first_element]['address']
ref = None
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone,
email, geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o,
sa_o, su_o, mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename), POST_DATA)
insert_data = []
if soup != None:
text = json.loads(soup.get_text())
for poi_data in text:
name = 'MOL'
code = 'humolfu'
postcode = poi_data['postcode'].strip()
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['address'])
city = clean_city(poi_data['city'])
branch = None
website = None
nonstop = None
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
original = poi_data['address']
ref = None
lat, lon = check_hu_boundary(poi_data['lat'], poi_data['lng'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, postcode)
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename), POST_DATA)
insert_data = []
if soup != None:
text = json.loads(soup.get_text())
for poi_data in text['results']:
name = 'OMV'
code = 'huomvfu'
postcode = poi_data['postcode'].strip()
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['address_l'])
city = clean_city(poi_data['town_l'])
branch = None
website = None
nonstop = None
if poi_data['open_hours'] is not None:
oho, ohc = clean_opening_hours(poi_data['open_hours'])
if oho == '00:00' and ohc == '24:00':
nonstop = True
oho, ohc = None, None
else:
oho, ohc = None, None
mo_o = oho
th_o = oho
we_o = oho
tu_o = oho
fr_o = oho
sa_o = oho
su_o = oho
mo_c = ohc
th_c = ohc
we_c = ohc
tu_c = ohc
fr_c = ohc
sa_c = ohc
su_c = ohc
original = poi_data['address_l']
ref = None
lat, lon = check_hu_boundary(poi_data['y'], poi_data['x'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, postcode)
if 'telnr' in poi_data and poi_data['telnr'] != '':
phone = clean_phone(poi_data['telnr'])
else:
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
xml = save_downloaded_xml(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
root = etree.fromstring(xml)
for e in root.findall('post'):
if e.find('ServicePointType').text == 'PM':
name = 'Posta'
code = 'hupostapo'
elif e.find('ServicePointType').text == 'CS':
name = 'Posta csomagautomata'
code = 'hupostacso'
elif e.find('ServicePointType').text == 'PP':
name = 'PostaPont'
code = 'hupostapp'
else:
logging.error('Non existing Posta type.')
postcode = e.get('zipCode')
street_tmp_1 = e.find('street/name').text.strip() if e.find(
'street/name').text is not None else None
street_tmp_2 = e.find('street/type').text.strip() if e.find(
'street/type').text is not None else None
if street_tmp_1 is None:
street = None
elif street_tmp_2 is None:
street = street_tmp_1
elif street_tmp_1 is not None and street_tmp_2 is not None:
street = '{} {}'.format(street_tmp_1, street_tmp_2)
else:
logging.error('Non handled state!')
housenumber = e.find('street/houseNumber').text.strip().lower(
) if e.find('street/houseNumber').text is not None else None
conscriptionnumber = None
city = clean_city(e.find('city').text)
branch = e.find('name').text if e.find(
'name').text is not None else None
website = None
nonstop = None
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
lat, lon = check_hu_boundary(
e.find('gpsData/WGSLat').text.replace(',', '.'),
e.find('gpsData/WGSLon').text.replace(',', '.'))
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(self.prefer_osm_postcode,
self.session, lat, lon,
postcode)
original = None
ref = None
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original, street,
housenumber, conscriptionnumber, ref, phone, email, geom,
nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o, mo_c, th_c,
we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
soup = save_downloaded_soup('{}'.format(self.link), os.path.join(self.download_cache, self.filename), None,
self.verify_link)
insert_data = []
if soup != None:
# parse the html using beautiful soap and store in variable `soup`
pattern = re.compile('^\s*var\s*places.*')
script = soup.find('script', text=pattern)
m = pattern.match(script.get_text())
data = m.group(0)
data = clean_javascript_variable(data, 'places')
text = json.loads(data)
for poi_data in text:
poi_data = poi_data['addresses'][0]
# Assign: code, postcode, city, name, branch, website, original, street, housenumber, conscriptionnumber, ref, geom
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['address'])
name = 'Rossmann'
code = 'hurossmche'
city = clean_city(poi_data['city'])
postcode = poi_data['zip'].strip()
branch = None
website = None
nonstop = False
if poi_data['business_hours']['monday'] is not None:
mo_o, mo_c = clean_opening_hours(poi_data['business_hours']['monday'])
else:
mo_o, mo_c = None, None
if poi_data['business_hours']['tuesday'] is not None:
th_o, th_c = clean_opening_hours(poi_data['business_hours']['tuesday'])
else:
th_o, th_c = None, None
if poi_data['business_hours']['wednesday'] is not None:
we_o, we_c = clean_opening_hours(poi_data['business_hours']['wednesday'])
else:
we_o, we_c = None, None
if poi_data['business_hours']['thursday'] is not None:
tu_o, tu_c = clean_opening_hours(poi_data['business_hours']['thursday'])
else:
tu_o, tu_c = None, None
if poi_data['business_hours']['friday'] is not None:
fr_o, fr_c = clean_opening_hours(poi_data['business_hours']['friday'])
else:
fr_o, fr_c = None, None
if poi_data['business_hours']['saturday'] is not None:
sa_o, sa_c = clean_opening_hours(poi_data['business_hours']['saturday'])
else:
sa_o, sa_c = None, None
if poi_data['business_hours']['sunday'] is not None:
su_o, su_c = clean_opening_hours(poi_data['business_hours']['sunday'])
else:
su_o, su_c = None, None
lat, lon = check_hu_boundary(poi_data['position'][0], poi_data['position'][1])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(self.prefer_osm_postcode, self.session, lat, lon, postcode)
original = poi_data['address']
ref = None
phone = None
email = None
insert_data.append(
[code, postcode, city, name, branch, website, original, street, housenumber, conscriptionnumber,
ref, phone, email, geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o, mo_c, th_c, we_c, tu_c,
fr_c, sa_c, su_c])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process_geom(self):
self.geom = check_geom(self.__lat, self.__lon)
def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
if soup != None:
text = json.loads(soup.get_text())
for poi_data in text:
# Assign: code, postcode, city, name, branch, website, original, street, housenumber, conscriptionnumber, ref, geom
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['address'])
if 'xpres' in poi_data['name']:
name = 'Spar Expressz'
code = 'husparexp'
elif 'INTER' in poi_data['name']:
name = 'Interspar'
code = 'husparint'
elif 'market' in poi_data['name']:
name = 'Spar'
code = 'husparsup'
else:
name = 'Spar'
code = 'husparsup'
poi_data['name'] = poi_data['name'].replace(
'INTERSPAR', 'Interspar')
poi_data['name'] = poi_data['name'].replace('SPAR', 'Spar')
ref_match = PATTERN_SPAR_REF.search(poi_data['name'])
ref = ref_match.group(
1).strip() if ref_match is not None else None
city = clean_city(poi_data['city'])
postcode = poi_data['zipCode'].strip()
branch = poi_data['name'].split('(')[0].strip()
website = poi_data['pageUrl'].strip()
nonstop = None
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
lat, lon = check_hu_boundary(poi_data['latitude'],
poi_data['longitude'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, postcode)
original = poi_data['address']
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
csv = save_downloaded_pd(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
if csv is not None:
csv[['Post code']] = csv[['Post code']].fillna('0000')
csv[['Post code']] = csv[['Post code']].astype(int)
csv[['Telephone']] = csv[['Telephone']].fillna('0')
csv[['Telephone']] = csv[['Telephone']].astype(int)
csv[['City']] = csv[['City']].fillna('')
csv[['Name']] = csv[['Name']].fillna('')
insert_data = []
poi_dict = csv.to_dict('records')
for poi_data in poi_dict:
if poi_data['Brand'] == 'Shell':
name = 'Shell'
code = 'hushellfu'
elif poi_data['Brand'] == 'Mobilpetrol':
name = 'Mobil Petrol'
code = 'humobpefu'
postcode = poi_data['Post code']
steet_tmp = poi_data['Address'].lower().split()
for i in range(0, len(steet_tmp) - 2):
steet_tmp[i] = steet_tmp[i].capitalize()
steet_tmp = ' '.join(steet_tmp)
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
steet_tmp)
if poi_data['City'] != '':
city = clean_city(poi_data['City'].title())
else:
if poi_data['Name'] != '':
city = clean_city(poi_data['Name'].title())
else:
city = None
branch = poi_data['Name'].strip()
website = None
if poi_data['24 Hour'] == True:
nonstop = True
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
else:
nonstop = False
mo_o = '06:00'
th_o = '06:00'
we_o = '06:00'
tu_o = '06:00'
fr_o = '06:00'
sa_o = '06:00'
su_o = '06:00'
mo_c = '22:00'
th_c = '22:00'
we_c = '22:00'
tu_c = '22:00'
fr_c = '22:00'
sa_c = '22:00'
su_c = '22:00'
original = poi_data['Address']
ref = None
lat, lon = check_hu_boundary(poi_data['GPS Latitude'],
poi_data['GPS Longitude'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, postcode)
if 'Telephone' in poi_data and poi_data['Telephone'] != '':
phone = clean_phone(str(poi_data['Telephone']))
else:
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
if soup != None:
# parse the html using beautiful soap and store in variable `soup`
pattern = re.compile('^\s*var\s*boltok_nyers.*')
script = soup.find('script', text=pattern)
m = pattern.match(script.get_text())
data = m.group(0)
data = clean_javascript_variable(data, 'boltok_nyers')
text = json.loads(data)
# for l in text:
# print ('postcode: {postcode}; city: {city}; address: {address}; alt_name: {alt_name}'.format(postcode=l['A_IRSZ'], city=l['A_VAROS'], address=l['A_CIM'], alt_name=l['P_NAME']))
for poi_data in text:
# Assign: code, postcode, city, name, branch, website, original, street, housenumber, conscriptionnumber, ref, geom
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['A_CIM'])
city = clean_city(poi_data['A_VAROS'])
postcode = poi_data['A_IRSZ'].strip()
branch = poi_data['P_NAME'].strip()
name = 'Príma' if 'Príma' in branch else 'CBA'
code = 'huprimacon' if 'Príma' in branch else 'hucbacon'
website = None
nonstop = None
mo_o = clean_opening_hours_2(
poi_data['PS_OPEN_FROM_1']
) if poi_data['PS_OPEN_FROM_1'] is not None else None
th_o = clean_opening_hours_2(
poi_data['PS_OPEN_FROM_2']
) if poi_data['PS_OPEN_FROM_2'] is not None else None
we_o = clean_opening_hours_2(
poi_data['PS_OPEN_FROM_3']
) if poi_data['PS_OPEN_FROM_3'] is not None else None
tu_o = clean_opening_hours_2(
poi_data['PS_OPEN_FROM_4']
) if poi_data['PS_OPEN_FROM_4'] is not None else None
fr_o = clean_opening_hours_2(
poi_data['PS_OPEN_FROM_5']
) if poi_data['PS_OPEN_FROM_5'] is not None else None
sa_o = clean_opening_hours_2(
poi_data['PS_OPEN_FROM_6']
) if poi_data['PS_OPEN_FROM_6'] is not None else None
su_o = clean_opening_hours_2(
poi_data['PS_OPEN_FROM_7']
) if poi_data['PS_OPEN_FROM_7'] is not None else None
mo_c = clean_opening_hours_2(
poi_data['PS_OPEN_TO_1']
) if poi_data['PS_OPEN_TO_1'] is not None else None
th_c = clean_opening_hours_2(
poi_data['PS_OPEN_TO_2']
) if poi_data['PS_OPEN_TO_2'] is not None else None
we_c = clean_opening_hours_2(
poi_data['PS_OPEN_TO_3']
) if poi_data['PS_OPEN_TO_3'] is not None else None
tu_c = clean_opening_hours_2(
poi_data['PS_OPEN_TO_4']
) if poi_data['PS_OPEN_TO_4'] is not None else None
fr_c = clean_opening_hours_2(
poi_data['PS_OPEN_TO_5']
) if poi_data['PS_OPEN_TO_5'] is not None else None
sa_c = clean_opening_hours_2(
poi_data['PS_OPEN_TO_6']
) if poi_data['PS_OPEN_TO_6'] is not None else None
su_c = clean_opening_hours_2(
poi_data['PS_OPEN_TO_7']
) if poi_data['PS_OPEN_TO_7'] is not None else None
original = poi_data['A_CIM']
lat, lon = check_hu_boundary(poi_data['PS_GPS_COORDS_LAT'],
poi_data['PS_GPS_COORDS_LNG'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, postcode)
ref = None
if 'PS_PUBLIC_TEL' in poi_data and poi_data[
'PS_PUBLIC_TEL'] != '':
phone = clean_phone(poi_data['PS_PUBLIC_TEL'])
else:
phone = None
if 'PS_PUBLIC_EMAIL' in poi_data and poi_data[
'PS_PUBLIC_EMAIL'] != '':
email = poi_data['PS_PUBLIC_EMAIL']
else:
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
if soup != None:
# parse the html using beautiful soap and store in variable `soup`
pattern = re.compile('var\s*markers\s*=\s*((.*\n)*\]\;)',
re.MULTILINE)
script = soup.find('script', text=pattern)
m = pattern.search(script.get_text())
data = m.group(0)
data = data.replace("'", '"')
data = clean_javascript_variable(data, 'markers')
text = json.loads(data)
for poi_data in text:
if poi_data['cim'] is not None and poi_data['cim'] != '':
postcode, city, street, housenumber, conscriptionnumber = extract_all_address(
poi_data['cim'])
name = 'Avia'
code = 'huaviafu'
branch = None
if city is None:
city = poi_data['title']
ref = poi_data['kutid'] if poi_data[
'kutid'] is not None and poi_data['kutid'] != '' else None
lat, lon = check_hu_boundary(poi_data['lat'], poi_data['lng'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, postcode)
website = '/toltoallomas/?id={}'.format(str(poi_data['kutid'])) if poi_data['kutid'] is not None and \
poi_data['kutid'] != '' else None
nonstop = None
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
original = poi_data['cim']
if 'tel' in poi_data and poi_data['tel'] != '':
phone = clean_phone(poi_data['tel'])
else:
phone = None
if 'email' in poi_data and poi_data['email'] != '':
email = clean_email(poi_data['email'])
else:
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
if soup != None:
# parse the html using beautiful soap and store in variable `soup`
# script = soup.find('div', attrs={'data-stores':True})
script = soup.find(attrs={'data-stores': True})
text = json.loads(script['data-stores'])
for poi_data in text:
# Assign: code, postcode, city, name, branch, website, original, street, housenumber, conscriptionnumber, ref, geom
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['address'])
city = clean_city(poi_data['city'])
branch = poi_data['name']
if 'xpres' in poi_data['name']:
name = 'Tesco Expressz'
code = 'hutescoexp'
elif 'xtra' in poi_data['name']:
name = 'Tesco Extra'
code = 'hutescoext'
else:
name = 'Tesco'
code = 'hutescosup'
website = poi_data['url']
nonstop = None
opening = json.loads(poi_data['opening'])
mo_o = opening['1'][0]
th_o = opening['2'][0]
we_o = opening['3'][0]
tu_o = opening['4'][0]
fr_o = opening['5'][0]
sa_o = opening['6'][0]
su_o = opening['0'][0]
mo_c = opening['1'][1]
th_c = opening['2'][1]
we_c = opening['3'][1]
tu_c = opening['4'][1]
fr_c = opening['5'][1]
sa_c = opening['6'][1]
su_c = opening['0'][1]
lat, lon = check_hu_boundary(poi_data['gpslat'],
poi_data['gpslng'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, None)
original = poi_data['address']
ref = None
if 'phone' in poi_data and poi_data['phone'] != '':
phone = clean_phone(poi_data['phone'])
else:
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
if soup != None:
text = json.loads(soup.get_text())
for poi_data in text['items']:
if poi_data['type'] == 'posta':
if 'mobilposta' in poi_data['name']:
name = 'Mobilposta'
code = 'hupostamp'
else:
name = 'Posta'
code = 'hupostapo'
elif poi_data['type'] == 'csekkautomata':
name = 'Posta csekkautomata'
code = 'hupostacse'
elif poi_data['type'] == 'postamachine':
name = 'Posta csomagautomata'
code = 'hupostacso'
elif poi_data['type'] == 'postapoint':
name = 'PostaPont'
code = 'hupostapp'
else:
logging.error('Non existing Posta type.')
postcode = poi_data['zipCode'].strip()
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['address'])
city = clean_city(poi_data['city'])
branch = poi_data['name']
website = None
nonstop = None
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
geom = check_geom(poi_data['lat'], poi_data['lng'])
original = poi_data['address']
ref = None
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
soup = save_downloaded_soup(
'{}'.format(self.link),
os.path.join(self.download_cache, self.filename))
insert_data = []
if soup != None:
text = json.loads(soup.get_text())
for poi_data in text:
name = 'Foxpost'
code = 'hufoxpocso'
postcode = poi_data['zip'].strip()
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['street'])
city = clean_city(poi_data['city'])
branch = poi_data['name']
website = None
nonstop = None
if poi_data['open']['hetfo'] is not None:
mo_o, mo_c = clean_opening_hours(poi_data['open']['hetfo'])
else:
mo_o, mo_c = None, None
if poi_data['open']['kedd'] is not None:
th_o, th_c = clean_opening_hours(poi_data['open']['kedd'])
else:
th_o, th_c = None, None
if poi_data['open']['szerda'] is not None:
we_o, we_c = clean_opening_hours(
poi_data['open']['szerda'])
else:
we_o, we_c = None, None
if poi_data['open']['csutortok'] is not None:
tu_o, tu_c = clean_opening_hours(
poi_data['open']['csutortok'])
else:
tu_o, tu_c = None, None
if poi_data['open']['pentek'] is not None:
fr_o, fr_c = clean_opening_hours(
poi_data['open']['pentek'])
else:
fr_o, fr_c = None, None
if poi_data['open']['szombat'] is not None:
sa_o, sa_c = clean_opening_hours(
poi_data['open']['szombat'])
else:
sa_o, sa_c = None, None
if poi_data['open']['vasarnap'] is not None:
su_o, su_c = clean_opening_hours(
poi_data['open']['vasarnap'])
else:
su_o, su_c = None, None
original = poi_data['address']
ref = None
lat, lon = check_hu_boundary(poi_data['geolat'],
poi_data['geolng'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon, postcode)
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)
def process(self):
'''
soup = save_downloaded_soup('{}'.format(self.link), os.path.join(self.download_cache, self.filename), POST_DATA)
insert_data = []
if soup != None:
text = json.loads(soup.get_text())
'''
with open(os.path.join(self.download_cache, self.filename), 'r') as f:
insert_data = []
text = json.load(f)
for poi_data in text:
street, housenumber, conscriptionnumber = extract_street_housenumber_better_2(
poi_data['cim'])
if 'Kulcs patika' not in poi_data['nev']:
name = poi_data['nev'].strip()
branch = None
else:
name = 'Kulcs patika'
branch = poi_data['nev'].strip()
code = 'hukulcspha'
website = poi_data['link'].strip(
) if poi_data['link'] is not None else None
nonstop = None
mo_o = None
th_o = None
we_o = None
tu_o = None
fr_o = None
sa_o = None
su_o = None
mo_c = None
th_c = None
we_c = None
tu_c = None
fr_c = None
sa_c = None
su_c = None
city = clean_city(poi_data['helyseg'])
lat, lon = check_hu_boundary(
poi_data['marker_position']['latitude'],
poi_data['marker_position']['longitude'])
geom = check_geom(lat, lon)
postcode = query_postcode_osm_external(
self.prefer_osm_postcode, self.session, lat, lon,
poi_data['irsz'].strip())
original = poi_data['cim']
ref = None
phone = None
email = None
insert_data.append([
code, postcode, city, name, branch, website, original,
street, housenumber, conscriptionnumber, ref, phone, email,
geom, nonstop, mo_o, th_o, we_o, tu_o, fr_o, sa_o, su_o,
mo_c, th_c, we_c, tu_c, fr_c, sa_c, su_c
])
if len(insert_data) < 1:
logging.warning('Resultset is empty. Skipping ...')
else:
df = pd.DataFrame(insert_data)
df.columns = POI_COLS
insert_poi_dataframe(self.session, df)