def save_info(db, visas, advisories, array_info):
LOGGER.info('Saving all countries information into SG table')
for country in visas:
try:
LOGGER.info(f'Saving information for {country}')
iso = find_iso_of_country(country)
visa = visas[country].get('visa')
advisory = advisories[country].get(
'advisory'
) #if the country doesn't have advisory info it throws an index error
info = {
"country_iso": iso,
"name": country,
"advisory": advisory,
"visa_info": visa
}
array_info.append(info)
LOGGER.info(f'Saving {country} into the SG table')
db.insert("SG", iso, country, advisory, visa)
LOGGER.success(
f'{country} was sucesfully saved to the database with the following information: {advisory}. {info}'
)
except KeyError: #if the country doesn't have advisory info
LOGGER.warning(
f'This country doesn’t have advisory info {country}')
iso = find_iso_of_country(country)
visa = visas[country].get('visa')
advisory = "Not available yet"
info = {
"country_iso": iso,
"name": country,
"advisory": advisory,
"visa_info": visa
}
array_info.append(info)
db.insert("SG", iso, country, advisory, visa)
for country in advisories: #countries that don't have visa info but have advisory info
if not country in visas:
LOGGER.warning(
f'This country doesn’t have advisory info {country}')
iso = find_iso_of_country(country)
visa_info = "Not available yet"
advisory = advisories[country].get('advisory')
info = {
"country_iso": iso,
"name": country,
"advisory": advisory,
"visa_info": visa_info
}
array_info.append(info)
db.insert("SG", iso, country, advisory, visa)
return array_info
def get_url_of_countries():
info = {}
LOGGER.info('Retrieving URL of all countries for United Kingdom advisory')
try:
#this is the link to the first page
url = 'https://www.gov.uk/foreign-travel-advice'
driver = create_driver()
driver.get(url)
#Selenium hands the page source to Beautiful Soup
soup=BeautifulSoup(driver.page_source, 'lxml')
#patter of the link to the country page that the href should match
countries_div = soup.findAll("div", {"class": "govuk-grid-column-two-thirds"})[1]
countries = countries_div.findAll('a')
#retrieving links for all countries
for country in countries:
country_name = country.text
country_iso = find_iso_of_country(country_name)
if(country_iso != ""): #Countries that don't have iso are not official counntries
href = country['href']
info[country_iso] = {"href":href}
LOGGER.success(f'URL of {country_name} was successfully retrieved')
except Exception as error_msg:
LOGGER.error(f'An error has occured while retrieving URL of countries for United Kingdom advisory because of the following error: {error_msg}')
finally:
driver.close()
driver.quit()
return info
def get_url_of_for_letter(dictionnary, letter):
try:
#this is the link to the first page
url = 'https://www.mfa.gov.sg/Where-Are-You-Travelling-To?letter={}'.format(
letter, sep='')
driver = create_driver()
driver.get(url)
#Selenium hands the page source to Beautiful Soup
soup = BeautifulSoup(driver.page_source, 'lxml')
#All countries for a given letter
countries = soup.findAll("a", {"class": "embassy-dropdown"})
# #retrieving links for all countries of the alphabet
for country in countries:
country_name = country.text.lstrip().rstrip()
country_iso = find_iso_of_country(country_name)
if (country_iso != ""
): #Countries that don't have iso are not official counntries
href = country['href']
dictionnary[country_name] = {"href": href}
finally:
driver.close()
driver.quit()
return dictionnary
def get_countries_sockets():
# Give the location of the file
loc = "./e_sockets.xlsx" #mac
# loc = "server/data/e_sockets.xlsx" #pc
# To open Workbook
wb = xlrd.open_workbook(loc)
sheet = wb.sheet_by_index(0)
sheet.cell_value(0, 0)
arrayOfInfo = []
#loop for every row and grab the values
for row in range(sheet.nrows):
country_name = sheet.cell_value(row, 0)
electric_potential = sheet.cell_value(row, 1)
frequency = sheet.cell_value(row, 2)
plug_type = sheet.cell_value(row, 3)
iso = find_iso_of_country(country_name)
info = {
"name":country_name,
"iso": iso,
"plug_type": plug_type,
"electric_potential": electric_potential,
"frequency": frequency
}
arrayOfInfo.append(info)
return arrayOfInfo
def save_to_MU():
LOGGER.info(f'Saving and parsing Mauritius into the databse')
driver = create_driver()
LOGGER.info('Begin parsing for Mauritius advisory')
try:
wiki_visa_url = wiki_visa_url_MU
wiki_visa_ob = wiki_visa_parser(wiki_visa_url, driver)
visas = wiki_visa_ob.visa_parser_table()
LOGGER.success(
'Parsing for Mauritius advisory has been successfully completed')
except Exception as error_msg:
LOGGER.error(
f'Error has occured while parsing for Mauritius advisory because of the following error: {error_msg}'
)
info = {}
array_info = []
# create an an sqlite_advisory object
db = Database("countries.sqlite")
db.drop_table("MU")
db.add_table("MU",
country_iso="text",
name="text",
advisory_text="text",
visa_info="text")
LOGGER.info('Saving Mauritius table into the database')
try:
for country in visas:
iso = find_iso_of_country(country)
if (iso != ""):
name = country
LOGGER.info(f'Saving {name}')
visa = visas[country].get(
'visa') #dictionary for visa info is country{visa:text}
advisory = "Not available yet"
info = {
"country_iso": iso,
"name": name,
"advisory": advisory,
"visa_info": visa
}
array_info.append(info)
print(name, " ", visa, " ", advisory)
db.insert("MU", iso, name, advisory, visa)
LOGGER.success(
f'{name} was sucessfully saved to the database with the following information: {visa}. {advisory}.'
)
LOGGER.success(
'Mauritius table successfully saved to the database')
except Exception as error_msg:
LOGGER.error(
f'An error has occured while saving Mauritius table to the database because of the following error: {error_msg}'
)
db.close_connection()
quit_driver(driver)
with open('./advisory-mu.json', 'w') as outfile:
json.dump(array_info, outfile)
def save_to_UK():
LOGGER.info("Begin parsing and saving for United Kingdom table...")
driver = create_driver()
LOGGER.info('Parsing the visa requirements of all countries for United Kingdom advisory')
try:
wiki_visa_url ="https://en.wikipedia.org/wiki/Visa_requirements_for_British_citizens"
wiki_visa_ob = wiki_visa_parser(wiki_visa_url,driver)
visas = wiki_visa_ob.visa_parser_table()
data = parse_all_countries_advisory()
LOGGER.success('Successfully parsed the visa requirements of all countries for United Kingdom advisory')
except Exception as error_msg:
LOGGER.error(f'An error has occured while retrieving the visa reuirements of all countries for United Kingdom advisory because of the following error: {error_msg}')
info = {}
array_info = []
# create an an sqlite_advisory object]
db = Database("countries.sqlite")
db.drop_table("GB")
db.add_table("GB", country_iso="text", name="text", advisory_text="text", visa_info="text")
LOGGER.info('Saving countries informations into the UK table')
try:
for country in visas:
iso = find_iso_of_country(country)
if(iso != ""):
try:
name = country
advisory = data[iso].get('advisory') #dictionary for the travel advisory is iso{advisory:text}
visa_info = visas[country].get('visa') #dictionary for visa info is country{visa:text}
info = {
"country_iso" : iso,
"name": name,
"advisory": advisory,
"visa_info": visa_info
}
array_info.append(info)
LOGGER.success(f"Saving {name} into the UK table with the following information: {visa_info}. {advisory}")
db.insert("GB",iso,name,advisory,visa_info)
LOGGER.success(f'{name} sucesfully saved to the database.')
except KeyError:
LOGGER.warning(f'This country doesn\'t have advisory info: {country}')
print("This country doesn't have advisory info: ",country)
LOGGER.info(f'Its ISO is {iso}')
print("Its ISO is: ",iso)
LOGGER.success('All countries have been succesfully saved into the UK table')
except Exception as error_msg:
LOGGER.error(f'An error has occured while saving countries into the UK table because of the following: {error_msg}')
db.close_connection()
with open('./advisory-uk.json', 'w') as outfile:
json.dump(array_info, outfile)
def get_country_traffic_side():
array_of_country_info = []
already_parsed = []
try:
# this is the link to the first page
url = 'https://www.worldstandards.eu/cars/list-of-left-driving-countries/'
driver = create_driver()
driver.get(url)
# Selenium hands the page source to Beautiful Soup
soup = BeautifulSoup(driver.page_source, 'html.parser')
table = soup.find('table')
tbody = table.find('tbody')
allRows = tbody.findAll('tr')
for country_row in allRows:
isHeader = country_row.find(
'th') != None #The header row should be discarded
if (not isHeader):
country = country_row.findAll('td')[0].text
traffic_side = country_row.findAll('td')[1].text
country_name_has_bracket = country.find('(')
if (country_name_has_bracket > -1
): #We want to remove the bracket from the country name
country = country[0:country_name_has_bracket]
country_iso = find_iso_of_country(country)
if (country_iso != ""):
if country_iso not in already_parsed: # Only parse the main traffic side of a country
if "left" in traffic_side:
traffic_side = "left"
else:
traffic_side = "right"
info = {
"country_iso": country_iso,
"country_name": country,
"traffic_side": traffic_side
}
already_parsed.append(country_iso)
array_of_country_info.append(info)
else:
print(
"The main land for this country is parsed already",
country)
return array_of_country_info
finally:
driver.close()
driver.quit()
def get_countries_cocainelaw():
LOGGER.info("Retrieving information for cocaine")
try:
# this is the link to the first page
url = 'https://en.wikipedia.org/wiki/Legal_status_of_cocaine'
driver = create_driver()
driver.get(url)
# Selenium hands the page source to Beautiful Soup
soup=BeautifulSoup(driver.page_source, 'html.parser')
# patter of the link to the country page that the href should match
table = soup.find('table', {'class':"wikitable"})
tbody = table.find('tbody')
table_rows = tbody.find_all('tr')
cocaine_info= {}
arrayCocaineInfo = {}
for tablerow in table_rows:
table_columns = tablerow.find_all('td')
if(len(table_columns)>0):
country_name= table_columns[0].text
cocaine_possession= table_columns[1].text
cocaine_possession= re.sub(r'\[\d*\]',' ',cocaine_possession.rstrip())
cocaine_sale= table_columns[2].text
cocaine_sale= re.sub(r'\[\d*\]',' ',cocaine_sale.rstrip())
cocaine_transport= table_columns[3].text
cocaine_transport= re.sub(r'\[\d*\]',' ',cocaine_transport.rstrip())
cocaine_cultivation= table_columns[4].text
cocaine_cultivation= re.sub(r'\[\d*\]',' ',cocaine_cultivation.rstrip())
country_iso = find_iso_of_country(country_name)
cocaine_info = {
"name":country_name,
"iso": country_iso,
"cocaine-possession": cocaine_possession,
"cocaine-sale": cocaine_sale,
"cocaine-transport": cocaine_transport,
"cocaine-cultivation": cocaine_cultivation
}
arrayCocaineInfo[country_iso] = cocaine_info
return arrayCocaineInfo
except Exception as error_msg:
LOGGER.error(f'An error has occured while retrieving information for cocaine because of the following error: {error_msg}')
finally:
driver.close()
driver.quit()
def get_countries_canabaislaw():
LOGGER.info("Retrieving information for canabais")
try:
# this is the link to the first page
url = 'https://en.wikipedia.org/wiki/Legality_of_cannabis'
driver = create_driver()
driver.get(url)
# Selenium hands the page source to Beautiful Soup
soup=BeautifulSoup(driver.page_source, 'html.parser')
# patter of the link to the country page that the href should match
table = soup.find('table', {'class':"wikitable"})
tbody = table.find('tbody')
table_rows = tbody.find_all('tr')
canabais_info= {}
arrayCanabaisInfo = {}
for tablerow in table_rows:
table_columns = tablerow.find_all('td')
if(len(table_columns)>0):
country_name= table_columns[0].text
recreational= table_columns[1].text
recreational= re.sub(r'\[\d*\]',' ',recreational.rstrip())
medical= table_columns[2].text
medical= re.sub(r'\[\d*\]',' ',medical.rstrip())
country_iso = find_iso_of_country(country_name)
canabais_info = {
"name":country_name,
"iso": country_iso,
"canabais-recreational": recreational,
"canabais-medical": medical
}
arrayCanabaisInfo[country_iso] = canabais_info
return arrayCanabaisInfo
except Exception as error_msg:
LOGGER.error(f'An error has occured while retrieving information for cannabis because of the followin error: {error_msg}')
finally:
driver.close()
driver.quit()
def get_url_of_countries():
info = {}
LOGGER.info('Retrieving URL of all countries for Vaccines table')
try:
#this is the link to the first page
driver = create_driver()
driver.get(vaccine_url)
#Selenium hands the page source to Beautiful Soup
soup = BeautifulSoup(driver.page_source, 'lxml')
#patter of the link to the country page that the href should match
countries_per_letter_array = soup.find_all("ul",
{"class": "list-bullet"})
for countries_per_letter in countries_per_letter_array:
# print(countries_div)
countries_given_letter_array = countries_per_letter.find_all('a')
#retrieving links for all countries
for country in countries_given_letter_array:
country_name = country.text
country_iso = find_iso_of_country(country_name)
if (
country_iso != ""
): #Countries that don't have iso are not official counntries
href = country['href']
info[country_iso] = {"href": href}
LOGGER.info(f' Retrieving URL of {country_name}, {href}')
except Exception as error_msg:
LOGGER.error(
f'Could not retrieve URLs of countries because of the following error: {error_msg}'
)
finally:
driver.close()
driver.quit()
return info
def get_additional_advisory_info_url():
url = 'https://travel.gc.ca/travelling/advisories'
#set up the headless chrome driver
driver = create_driver()
driver.get(url)
soup = BeautifulSoup(driver.page_source, 'lxml')
table = soup.find('table')
table_body = table.find('tbody')
table_rows = table_body.find_all('tr', attrs={'class': 'gradeX'})
additional_advisory = {}
for row in table_rows:
cols = row.find_all('td')
country = cols[0].text
iso = find_iso_of_country(country)
advisory = cols[2].text
additional_advisory[iso] = {
'country_name': country,
'advisory_text': advisory
}
quit_driver(driver)
return additional_advisory