def save_downloaded_soup(link,
file,
filetype,
post_data=None,
verify=config.get_download_verify_link(),
headers=None):
if config.get_download_use_cached_data() is True and os.path.isfile(file):
soup = readfile(file, filetype)
else:
if link is not None:
soup = download_content(link, verify, post_data, headers)
if soup is not None:
logging.info('We got content, write to file.')
if not os.path.exists(config.get_directory_cache_url()):
os.makedirs(config.get_directory_cache_url())
with open(file, mode='w', encoding='utf-8') as code:
if filetype == FileType.html:
soup = BeautifulSoup(soup, 'html.parser')
code.write(str(soup.prettify()))
elif filetype == FileType.xml:
soup = BeautifulSoup(soup,
'lxml',
from_encoding='utf-8')
logging.debug('original encoding: %s',
soup.original_encoding)
code.write(str(soup.prettify()))
elif filetype == FileType.csv or filetype == FileType.json:
code.write(str(soup))
else:
logging.error('Unexpected type to write: %s', filetype)
else:
if os.path.exists(file):
logging.info(
'The %s link returned error code other than 200 but there is an already downloaded file. Try to open it.',
link)
soup = readfile(file, filetype)
else:
logging.warning(
'Skipping dataset: %s. There is not downloadable URL, nor already downbloaded file.',
link)
else:
if os.path.exists(file):
soup = readfile(file, filetype)
if filetype == FileType.html:
soup = BeautifulSoup(soup, 'html.parser')
elif filetype == FileType.xml:
soup = BeautifulSoup(soup, 'lxml')
logging.info(
'Using file only: %s. There is not downloadable URL only just the file. Do not forget to update file manually!',
file)
else:
logging.warning(
'Cannot use download and file: %s. There is not downloadable URL, nor already downbloaded file.',
file)
return soup
def save_downloaded_pd(link, file, verify=config.get_download_verify_link()):
if config.get_download_use_cached_data() == True and os.path.isfile(file):
df = pd.read_csv(file)
else:
df = pd.read_csv(link, encoding='UTF-16', sep='\t')
if df is not None:
if not os.path.exists(config.get_directory_cache_url()):
os.makedirs(config.get_directory_cache_url())
df.to_csv(file)
else:
logging.warning('Skipping dataset.')
return df
def save_downloaded_xml(link, file, verify=config.get_download_verify_link()):
if config.get_download_use_cached_data() == True and os.path.isfile(file):
with open(file, 'rb') as content_file:
page = content_file.read()
else:
page = download_xml(link, verify)
if page != None:
if not os.path.exists(config.get_directory_cache_url()):
os.makedirs(config.get_directory_cache_url())
with open(file, mode='wb') as code:
code.write(page)
else:
logging.warning('Skipping dataset.')
return page
def save_downloaded_soup(link,
file,
post_data=None,
verify=config.get_download_verify_link()):
if config.get_download_use_cached_data() == True and os.path.isfile(file):
with open(file, 'r') as content_file:
soup = BeautifulSoup(content_file.read(), 'html.parser')
else:
soup = download_soup(link, verify, post_data)
if soup != None:
if not os.path.exists(config.get_directory_cache_url()):
os.makedirs(config.get_directory_cache_url())
with open(file, mode="w", encoding="utf8") as code:
code.write(str(soup))
else:
logging.warning('Skipping dataset: {}.'.format(link))
return soup
def save_downloaded_pd(link,
file,
verify=config.get_download_verify_link(),
headers=None):
if config.get_download_use_cached_data() is True and os.path.isfile(file):
df = pd.read_csv(file)
else:
if link is not None:
cvs = download_content(link, verify, None, None, 'utf-16')
if cvs is not None:
logging.info('We got content, write to file.')
if not os.path.exists(config.get_directory_cache_url()):
os.makedirs(config.get_directory_cache_url())
with open(file, mode='w', encoding='utf-8') as code:
code.write(cvs)
df = pd.read_csv(file, encoding='UTF-8', sep='\t', skiprows=0)
else:
if os.path.exists(file):
logging.info(
'The %s link returned error code other than 200 but there is an already downloaded file. Try to open it.',
link)
df = pd.read_csv(file,
encoding='UTF-8',
sep='\t',
skiprows=0)
else:
logging.warning(
'Skipping dataset: %s. There is not downloadable URL, nor already downbloaded file.',
link)
else:
if os.path.exists(file):
df = pd.read_csv(file, encoding='UTF-8', sep='\t', skiprows=0)
logging.info(
'Using file only: %s. There is not downloadable URL only just the file. Do not forget to update file manually!',
file)
else:
logging.warning(
'Cannot use download and file: %s. There is not downloadable URL, nor already downbloaded file.',
file)
return df
def constains(self):
# self.link = 'https://www.magnetbank.hu/kapcsolat/fiokkereso'
self.link = os.path.join(config.get_directory_cache_url(),
'hu_magnet_bank.json')
self.tags = {
'brand': 'MagNet Bank',
'brand:wikidata': 'Q17379757',
'bic': 'HBWEHUHB',
'brand:wikipedia': 'hu:MagNet Bank',
'operator': 'MagNet Magyar Közösségi Bank Zrt.',
'operator:addr': '1062 Budapest, Andrássy út 98.',
'contact:fax': '+36 1 428 8889',
'ref:HU:company': '01 10 046111',
'ref:vatin': 'HU14413591',
'ref:vatin:hu': '14413591-4-44',
}
self.filetype = FileType.json
self.filename = '{}.{}'.format(self.__class__.__name__,
self.filetype.name)
def constains(self):
self.link = os.path.join(config.get_directory_cache_url(),
'hu_mobiliti_ev.csv')
self.tags = {
'amenity': 'charging_station',
'authentication:app': 'yes',
'authentication:none': 'yes',
'authentication:membership_card': 'yes',
'operator': 'NKM Mobilitás Kft.',
'operator:addr': '1081 Budapest, II. János Pál pápa tér 20.',
'fee': 'yes',
'parking:fee': 'no',
'opening_hours': '24/7',
'ref:vatin': 'HU23443486',
'ref:vatin:hu': '23443486-2-42',
'ref:HU:company': '01-09-965868',
'contact:website': 'https://www.mobiliti.hu/emobilitas',
'contact:email': '*****@*****.**',
'contact:phone': '+36 62 565 758',
}
self.filetype = FileType.csv
self.filename = '{}.{}'.format(self.__class__.__name__,
self.filetype.name)
def constains(self):
self.link = os.path.join(config.get_directory_cache_url(),
'hu_mol_plugee_ev.csv')
self.tags = {
'amenity': 'charging_station',
'authentication:app': 'yes',
'authentication:none': 'yes',
'brand': 'MOL',
'operator': 'MOL Nyrt.',
'operator:addr': '1117 Budapest, Október huszonharmadika utca 18.',
'fee': 'yes',
'parking:fee': 'no',
'opening_hours': '24/7',
'ref:vatin': 'HU10625790',
'ref:vatin:hu': '10625790-4-44',
'ref:HU:company': '01-10-041683',
'contact:email': '*****@*****.**',
'contact:phone': '+36 1 998 9888',
'contact:website': 'https://molplugee.hu/',
'motorcar': 'yes'
}
self.filetype = FileType.csv
self.filename = '{}.{}'.format(self.__class__.__name__,
self.filetype.name)
def import_poi_data_module(module: str):
"""Process all data provider modules enabled in app.conf and write to the database
Args:
module (str): Name of module to run
"""
try:
db = POIBase('{}://{}:{}@{}:{}/{}'.format(
config.get_database_type(), config.get_database_writer_username(),
config.get_database_writer_password(),
config.get_database_writer_host(),
config.get_database_writer_port(),
config.get_database_poi_database()))
pgsql_pool = db.pool
session_factory = sessionmaker(pgsql_pool)
Session = scoped_session(session_factory)
session = Session()
module = module.strip()
logging.info('Processing %s module ...', module)
if module == 'hu_kh_bank':
from osm_poi_matchmaker.dataproviders.hu_kh_bank import hu_kh_bank
work = hu_kh_bank(
session, config.get_directory_cache_url(), True,
os.path.join(config.get_directory_cache_url(),
'hu_kh_bank.json'), 'K&H Bank')
insert_type(session, work.types())
work.process()
work = hu_kh_bank(
session, config.get_directory_cache_url(), True,
os.path.join(config.get_directory_cache_url(),
'hu_kh_atm.json'), 'K&H Bank ATM')
work.process()
elif module == 'hu_cib_bank':
from osm_poi_matchmaker.dataproviders.hu_cib_bank import hu_cib_bank
work = hu_cib_bank(
session, config.get_directory_cache_url(), True,
os.path.join(config.get_directory_cache_url(),
'hu_cib_bank.json'), 'CIB Bank')
insert_type(session, work.types())
work.process()
work = hu_cib_bank(
session, config.get_directory_cache_url(), True,
os.path.join(config.get_directory_cache_url(),
'hu_cib_atm.json'), 'CIB Bank ATM')
work.process()
elif module == 'hu_posta_json':
# Old code that uses JSON files
from osm_poi_matchmaker.dataproviders.hu_posta_json import hu_posta_json
# We only using csekkautomata since there is no XML from another data source
work = hu_posta_json(
session,
'https://www.posta.hu/szolgaltatasok/posta-srv-postoffice/rest/postoffice/list?searchField=&searchText=&types=csekkautomata',
config.get_directory_cache_url(), 'hu_postacsekkautomata.json')
work.process()
else:
mo = dataproviders_loader.import_module(
'dataproviders.{0}'.format(module), module)
work = mo(session, config.get_directory_cache_url())
insert_type(session, work.types())
work.process()
work.export_list()
except Exception as err:
logging.error(err)
logging.exception('Exception occurred')