def mexico_all_links(driver):
#the mexican website provides advisory in spanish
#we can display the link
url = 'https://guiadelviajero.sre.gob.mx/'
driver.get(url)
soup = BeautifulSoup(driver.page_source, 'lxml')
reg = regex.compile(r'\/103-ficha-de-paises\/')
a = soup.findAll('a', attrs={'href': reg})
links = {}
iso_es = get_iso_es()
LOGGER.info(
f'Retrieving the URLs for all countries for the Mexico advisory')
for att in a:
try:
name = att.text.strip()
iso = iso_es[name]
href = 'https://guiadelviajero.sre.gob.mx' + att['href']
href = '<a href =\'' + href + '\'>Mexican Government Webesite</a>'
links[iso] = {
'advisory_text': href,
'country_iso': iso,
'name': name
}
Logger.success(f'The URL for {name} was successfully retrieved')
LOGGER.success(
'Successfully retrieved the URLs for all countries of the Mexican advisory'
)
except Exception as error_msg:
LOGGER.warning(
f"This country's iso was not found for {name} because of the following error: {error_msg}"
)
#get the visa for mexico like for other countries from wikipedia
LOGGER.info(
'Parsing visa requirements for all countries for the Mexican advisory')
try:
wiki_visa_ob_MX = wiki_visa_parser(wiki_visa_url_MX, driver)
visas = wiki_visa_ob_MX.visa_parser_table()
visas = replace_key_by_iso(visas)
LOGGER.success(
'Successfully parsed all countries for the Mexican advisory')
except Exception as error_msg:
LOGGER.error(
f'Was not successful in parsing visa requirements for Mexican advisory because of the following error: {error_msg}'
)
data = {}
for key in visas:
try:
data[key] = links[key]
info = data[key]
info['visa-info'] = visas[key].get('visa-info')
except Exception as error_msg:
LOGGER.warning(
f'The following iso was not found: {key} because of the following error: {error_msg}'
)
return links
def find_all_ireland():
LOGGER.info("Begin parsing and saving for Ireland...")
my_driver = create_driver()
all_url = find_all_url(my_driver)
data = find_all_iso(all_url)
LOGGER.info(
'Parsing visa requirements for all countries for the Ireland advisory')
try:
wiki_visa_ob = wiki_visa_parser(
"https://en.wikipedia.org/wiki/Visa_requirements_for_Irish_citizens",
my_driver)
visas = wiki_visa_ob.visa_parser_table()
except Exception as error_msg:
LOGGER.error(
f'An error has occured while getting the visa requirements for Ireland advisory because of the following error: {error_msg}'
)
for country in data:
c = data[country]
url = c['href']
my_driver.implicitly_wait(5)
my_driver.get(url)
soup = BeautifulSoup(my_driver.page_source, 'lxml')
c['visa-info'] = get_one_info(url, 'visa/passport', my_driver, soup)
c['advisory-text'] = get_one_advisory(url, my_driver, soup)
c['name'] = country
if c['visa-info'] == '':
c['visa-info'] = get_one_info(url, 'Entry requirements', my_driver,
soup)
iso = c['country-iso']
#handling some exceptions, had to do research
if iso == 'AI':
c['visa-info'] = 'Visa not required for 3 months'
elif iso == 'BM':
c['visa-info'] = 'Visa not required for 21 days (extendable)'
elif iso == 'MQ':
iso = 'FR'
elif iso == 'MS':
c['visa-info'] = 'Visa not required for 6 months'
elif iso == 'RE':
iso = 'FR'
else:
try:
c['visa-info'] = visas[country].get(
'visa') + "<br>" + c['visa-info']
except Exception as error_msg:
print(c, error_msg)
LOGGER.warning(f'Error message: {error_msg}')
#dump the data into js to be deleted later
quit_driver(my_driver)
with open('./advisory-ie.json', 'w') as outfile:
json.dump(data, outfile)
save_into_db(data)
#find_all_ireland()
def save_to_MU():
LOGGER.info(f'Saving and parsing Mauritius into the databse')
driver = create_driver()
LOGGER.info('Begin parsing for Mauritius advisory')
try:
wiki_visa_url = wiki_visa_url_MU
wiki_visa_ob = wiki_visa_parser(wiki_visa_url, driver)
visas = wiki_visa_ob.visa_parser_table()
LOGGER.success(
'Parsing for Mauritius advisory has been successfully completed')
except Exception as error_msg:
LOGGER.error(
f'Error has occured while parsing for Mauritius advisory because of the following error: {error_msg}'
)
info = {}
array_info = []
# create an an sqlite_advisory object
db = Database("countries.sqlite")
db.drop_table("MU")
db.add_table("MU",
country_iso="text",
name="text",
advisory_text="text",
visa_info="text")
LOGGER.info('Saving Mauritius table into the database')
try:
for country in visas:
iso = find_iso_of_country(country)
if (iso != ""):
name = country
LOGGER.info(f'Saving {name}')
visa = visas[country].get(
'visa') #dictionary for visa info is country{visa:text}
advisory = "Not available yet"
info = {
"country_iso": iso,
"name": name,
"advisory": advisory,
"visa_info": visa
}
array_info.append(info)
print(name, " ", visa, " ", advisory)
db.insert("MU", iso, name, advisory, visa)
LOGGER.success(
f'{name} was sucessfully saved to the database with the following information: {visa}. {advisory}.'
)
LOGGER.success(
'Mauritius table successfully saved to the database')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while saving Mauritius table to the database because of the following error: {error_msg}'
)
db.close_connection()
quit_driver(driver)
with open('./advisory-mu.json', 'w') as outfile:
json.dump(array_info, outfile)
def save_to_united_states():
LOGGER.info("Begin parsing and saving for United States table...")
driver = create_driver()
data = {} #Used to store of all the parsed data of each country
name_to_advisories = {} #Stores the names and associated advisories
LOGGER.info(
f'Retrieving visa requirements for all countries for the United States advisory'
)
name_advisory = get_name_and_advisory_of_countries()
wiki_visa_url = "https://en.wikipedia.org/wiki/Visa_requirements_for_United_States_citizens"
wiki_visa_ob = wiki_visa_parser(wiki_visa_url, driver)
visas = wiki_visa_ob.visa_parser_table()
LOGGER.success(
'Successfully retrieved visa requirements for all countries for the United States advisory'
)
for name in sorted(name_advisory.keys(
)): #Sorts the dictionary containing names and advisories
name_to_advisories[name] = name_advisory[name]
counter_country = 0
for country in name_to_advisories: #iterates through name_to_advisories to retrieve advisories
driver.implicitly_wait(5)
name = country
advisory = name_to_advisories[country]
visa_text = ""
for countryVisa in visas: # iterates through list of visas to retrieve visas
if (countryVisa == country):
visa_text = visas[countryVisa].get('visa')
del visas[countryVisa]
break
country_iso = "na"
data[name] = {
'country-iso': country_iso,
'name': name,
'advisory-text': advisory,
'visa-info': visa_text
}
if ((counter_country % 50) == 0):
quit_driver(driver)
driver = create_driver()
counter_country += 1
data = find_all_iso(data) #Sets iso for each country
with open('./advisory-us.json', 'w') as outfile:
json.dump(data, outfile)
save_into_db(data)
def save_to_UK():
LOGGER.info("Begin parsing and saving for United Kingdom table...")
driver = create_driver()
LOGGER.info('Parsing the visa requirements of all countries for United Kingdom advisory')
try:
wiki_visa_url ="https://en.wikipedia.org/wiki/Visa_requirements_for_British_citizens"
wiki_visa_ob = wiki_visa_parser(wiki_visa_url,driver)
visas = wiki_visa_ob.visa_parser_table()
data = parse_all_countries_advisory()
LOGGER.success('Successfully parsed the visa requirements of all countries for United Kingdom advisory')
except Exception as error_msg:
LOGGER.error(f'An error has occured while retrieving the visa reuirements of all countries for United Kingdom advisory because of the following error: {error_msg}')
info = {}
array_info = []
# create an an sqlite_advisory object]
db = Database("countries.sqlite")
db.drop_table("GB")
db.add_table("GB", country_iso="text", name="text", advisory_text="text", visa_info="text")
LOGGER.info('Saving countries informations into the UK table')
try:
for country in visas:
iso = find_iso_of_country(country)
if(iso != ""):
try:
name = country
advisory = data[iso].get('advisory') #dictionary for the travel advisory is iso{advisory:text}
visa_info = visas[country].get('visa') #dictionary for visa info is country{visa:text}
info = {
"country_iso" : iso,
"name": name,
"advisory": advisory,
"visa_info": visa_info
}
array_info.append(info)
LOGGER.success(f"Saving {name} into the UK table with the following information: {visa_info}. {advisory}")
db.insert("GB",iso,name,advisory,visa_info)
LOGGER.success(f'{name} sucesfully saved to the database.')
except KeyError:
LOGGER.warning(f'This country doesn\'t have advisory info: {country}')
print("This country doesn't have advisory info: ",country)
LOGGER.info(f'Its ISO is {iso}')
print("Its ISO is: ",iso)
LOGGER.success('All countries have been succesfully saved into the UK table')
except Exception as error_msg:
LOGGER.error(f'An error has occured while saving countries into the UK table because of the following: {error_msg}')
db.close_connection()
with open('./advisory-uk.json', 'w') as outfile:
json.dump(array_info, outfile)
def save_to_australia():
LOGGER.info("Begin parsing and saving for Australia table...")
url = get_url_of_countries(
) #this function create its own driver -- to change
data = {}
driver = create_driver()
try:
LOGGER.info(
'Parsing visa requirements for all countries for the Australian advisory'
)
wiki_visa_url = 'https://en.wikipedia.org/wiki/Visa_requirements_for_Australian_citizens'
wiki_visa_ob = wiki_visa_parser(wiki_visa_url, driver)
wiki_visa = wiki_visa_ob.visa_parser_table()
except Exception as error_msg:
LOGGER.error(
f'An error has occured while retrieving the visa requirements for all countries for the Australian advisory because of following error: {error_msg}'
)
for country in url:
driver.implicitly_wait(5)
name = country
href = url[country].get('href')
advisory_text = url[country].get('advisory-text')
link = "https://smartraveller.gov.au{}".format(href, sep='')
additional_advisory = get_additional_advisory(link, driver)
advisory_text = advisory_text + additional_advisory
LOGGER.info(f"Begin parsing {name} to insert into AU table")
visa_info = parse_a_country(link, driver, 'Visas')
LOGGER.success(
f"The following information was retrieved for {name}: {visa_info}. {advisory_text}"
)
if (visa_info == ''):
try:
visa_info = wiki_visa[name].get('visa') + "<br>" + visa_info
except:
LOGGER.warning(f"No visa info for {name}")
country_iso = "na"
data[name] = {
'country-iso': country_iso,
'name': name,
'advisory-text': advisory_text,
'visa-info': visa_info
}
driver.quit()
data = find_all_iso(data)
save_into_db(data)
def save_to_new_zealand():
LOGGER.info("Begin parsing and saving for New Zealand table...")
driver = create_driver()
data = {} #Used to store of all the parsed data of each country
url = get_url_of_countries_nz(driver) #this function create its own driver -- to change
LOGGER.info('Retrieving visa requirements for New Zealand advisory')
try:
wiki_visa_url = "https://en.wikipedia.org/wiki/Visa_requirements_for_New_Zealand_citizens"
wiki_visa_ob = wiki_visa_parser(wiki_visa_url,driver)
visas = wiki_visa_ob.visa_parser_table()# Used to acquire visa info of each country
LOGGER.success('Succesfully retrieved visa requirements of all countries for New Zealand advisory')
except Exception as error_msg:
LOGGER.error(f'An error has occured while retrieving visa requirement for New Zealand adviosry because of the following error: {error_msg}')
counter_country = 0
for country in url: #iterates through urls to retrieve advisory information
driver.implicitly_wait(5)
name = country
href = url[country].get("href")
link = "https://safetravel.govt.nz/{}".format(href,sep='')
advisory = parse_a_country_advisory(link,driver)
visa_text= ""
for countryVisa in visas: # iterates through list of visas to retrieve visas
if(countryVisa == country):
visa_text = visas[countryVisa].get('visa')
del visas[countryVisa]
break;
country_iso = "na"
data[name] = {'country-iso':country_iso,'name':name,'advisory-text':advisory,'visa-info':visa_text}
if ((counter_country%50) == 0):
quit_driver(driver)
driver = create_driver()
counter_country += 1
data = find_all_iso(data)#Sets iso for each country
with open('./advisory-nz.json', 'w') as outfile:
json.dump(data, outfile)
save_into_db(data)
def save_to_SG():
LOGGER.info(f'Saving Singapore into the databse')
driver = create_driver()
LOGGER.info(
'Parsing visa requirments for all countries into the Singapore table')
try:
wiki_visa_url = wiki_visa_url_SG
wiki_visa_ob = wiki_visa_parser(wiki_visa_url, driver)
visas = wiki_visa_ob.visa_parser_table()
LOGGER.success(
'Visa requirements have been succesfully parsed for the Singapore table'
)
except Exception as error_msg:
LOGGER.error(
f'An error has occured whilse parsing for visa requirements because of the following error: {error_msg}'
)
advisories = parse_all_countries_advisories()
array_info = []
# create an an sqlite_advisory object
db = Database("countries.sqlite")
db.drop_table("SG")
db.add_table("SG",
country_iso="text",
name="text",
advisory_text="text",
visa_info="text")
array_info = save_info(db, visas, advisories, array_info)
db.close_connection()
LOGGER.success(f'Singapore was sucesfully saved to the database')
quit_driver(driver)
with open('./advisory-sg.json', 'w') as outfile:
json.dump(array_info, outfile)
def save_to_central_america():
LOGGER.info("Begin parsing and saving for Central America...")
#create driver
driver = create_driver()
#Mexico
data_MX = mexico_all_links(driver)
LOGGER.info("Saving Mexico to Central America")
try:
save_into_db_MX('MX', data_MX)
LOGGER.success("MX successfully saved into the databse")
except Exception as error_msg:
LOGGER.error(
f'MX was not successfully saved into the database because of the following error: {error_msg}'
)
#create obj driver and set belize as first url
driver = create_driver()
LOGGER.info(f'Beginning parsing for Belize')
try:
wiki_visa = wiki_visa_parser(wiki_visa_url_BZ, driver)
visa_BZ = wiki_visa.visa_parser_table()
visa_BZ = replace_key_by_iso(visa_BZ)
LOGGER.success(f'Following data was retrieved: {visa_BZ}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Belize because of the following error: {error_msg}'
)
#Dominica
driver.close()
driver = create_driver()
LOGGER.info(f'Beginning parsing for Dominica')
try:
wiki_visa = wiki_visa_parser(wiki_visa_url_DM, driver)
visa_DM = wiki_visa.visa_parser_table()
visa_DM = replace_key_by_iso(visa_DM)
LOGGER.success(f'Following data was retrieved: {visa_DM}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Dominica because of the following error: {error_msg}'
)
#Dominican Republic
driver.close()
driver = create_driver()
LOGGER.info(f'Beginning parsing for Dominican Republic')
try:
wiki_visa = wiki_visa_parser(wiki_visa_url_DO, driver)
visa_DO = wiki_visa.visa_parser_table()
visa_DO = replace_key_by_iso(visa_DO)
LOGGER.success(f'Following data was retrieved: {visa_DO}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Dominican Republic because of the following error" {error_msg}'
)
#Panama
driver.close()
driver = create_driver()
LOGGER.info(f'Beginning parsing for Panama')
try:
wiki_visa = wiki_visa_parser(wiki_visa_url_PA, driver)
visa_PA = wiki_visa.visa_parser_table()
visa_PA = replace_key_by_iso(visa_PA)
LOGGER.success(f'Following data was retrieved: {visa_PA}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Panama because of the following error: {error_msg}'
)
driver.quit()
#save the data into the DB
save_into_db("BZ", visa_BZ)
save_into_db("DM", visa_DM)
save_into_db("DO", visa_DO)
save_into_db("PA", visa_PA)
def save_to_caribbea():
#Antigua and Barbuda
LOGGER.info(f'Beginning parsing for Antigua and Barbuda')
try:
driver = create_driver()
wiki_visa = wiki_visa_parser(wiki_visa_url_AG, driver)
visa_AG = wiki_visa.visa_parser_table()
visa_AG = replace_key_by_iso(visa_AG)
LOGGER.success(f'Following data was retrieved: {visa_AG}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Antigua and Barbuda because of the following error: {error_msg}'
)
driver.close()
# Barbados
LOGGER.info(f'Beginning parsing for Barbados')
try:
driver = create_driver()
wiki_visa = wiki_visa_parser(wiki_visa_url_BB, driver)
visa_BB = wiki_visa.visa_parser_table()
visa_BB = replace_key_by_iso(visa_BB)
LOGGER.success(f'Following data was retrieved: {visa_BB}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Barbados because of the following error: {error_msg}'
)
driver.close()
#Bahamas
LOGGER.info(f'Beginning parsing for Bahamas')
try:
driver = create_driver()
wiki_visa = wiki_visa_parser(wiki_visa_url_BS, driver)
visa_BS = wiki_visa.visa_parser_table()
visa_BS = replace_key_by_iso(visa_BS)
LOGGER.success(f'Following data was retrieved: {visa_BS}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Bahamas because of the following error: {error_msg}'
)
driver.close()
#Grenada
LOGGER.info(f'Beginning parsing for Grenada')
try:
driver = create_driver()
wiki_visa = wiki_visa_parser(wiki_visa_url_GD, driver)
visa_GD = wiki_visa.visa_parser_table()
visa_GD = replace_key_by_iso(visa_GD)
LOGGER.success(f'Following data was retrieved: {visa_GD}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Grenada because of the following error: {error_msg}'
)
driver.close()
#Jamaica
LOGGER.info(f'Beginning parsing for Jamaica')
try:
driver = create_driver()
wiki_visa = wiki_visa_parser(wiki_visa_url_JM, driver)
visa_JM = wiki_visa.visa_parser_table()
visa_JM = replace_key_by_iso(visa_JM)
LOGGER.success(f'Following data was retrieved: {visa_JM}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Jamaica because of the following error: {error_msg}'
)
driver.close()
#Trinidad and Tobago
LOGGER.info(f'Beginning parsing for Trinidad and Tobago')
try:
driver = create_driver()
wiki_visa = wiki_visa_parser(wiki_visa_url_TT, driver)
visa_TT = wiki_visa.visa_parser_table()
visa_TT = replace_key_by_iso(visa_TT)
LOGGER.success(f'Following data was retrieved: {visa_TT}')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while parsing for Trinidad and Tobago because of the following error: {error_msg}'
)
driver.quit()
save_into_db("AG", visa_AG)
save_into_db("BB", visa_BB)
save_into_db("BS", visa_BS)
save_into_db("GD", visa_GD)
save_into_db("JM", visa_JM)
save_into_db("TT", visa_TT)
