def save_info(db, visas, advisories, array_info): LOGGER.info('Saving all countries information into SG table') for country in visas: try: LOGGER.info(f'Saving information for {country}') iso = find_iso_of_country(country) visa = visas[country].get('visa') advisory = advisories[country].get( 'advisory' ) #if the country doesn't have advisory info it throws an index error info = { "country_iso": iso, "name": country, "advisory": advisory, "visa_info": visa } array_info.append(info) LOGGER.info(f'Saving {country} into the SG table') db.insert("SG", iso, country, advisory, visa) LOGGER.success( f'{country} was sucesfully saved to the database with the following information: {advisory}. {info}' ) except KeyError: #if the country doesn't have advisory info LOGGER.warning( f'This country doesn’t have advisory info {country}') iso = find_iso_of_country(country) visa = visas[country].get('visa') advisory = "Not available yet" info = { "country_iso": iso, "name": country, "advisory": advisory, "visa_info": visa } array_info.append(info) db.insert("SG", iso, country, advisory, visa) for country in advisories: #countries that don't have visa info but have advisory info if not country in visas: LOGGER.warning( f'This country doesn’t have advisory info {country}') iso = find_iso_of_country(country) visa_info = "Not available yet" advisory = advisories[country].get('advisory') info = { "country_iso": iso, "name": country, "advisory": advisory, "visa_info": visa_info } array_info.append(info) db.insert("SG", iso, country, advisory, visa) return array_info
def get_url_of_countries(): info = {} LOGGER.info('Retrieving URL of all countries for United Kingdom advisory') try: #this is the link to the first page url = 'https://www.gov.uk/foreign-travel-advice' driver = create_driver() driver.get(url) #Selenium hands the page source to Beautiful Soup soup=BeautifulSoup(driver.page_source, 'lxml') #patter of the link to the country page that the href should match countries_div = soup.findAll("div", {"class": "govuk-grid-column-two-thirds"})[1] countries = countries_div.findAll('a') #retrieving links for all countries for country in countries: country_name = country.text country_iso = find_iso_of_country(country_name) if(country_iso != ""): #Countries that don't have iso are not official counntries href = country['href'] info[country_iso] = {"href":href} LOGGER.success(f'URL of {country_name} was successfully retrieved') except Exception as error_msg: LOGGER.error(f'An error has occured while retrieving URL of countries for United Kingdom advisory because of the following error: {error_msg}') finally: driver.close() driver.quit() return info
def get_url_of_for_letter(dictionnary, letter): try: #this is the link to the first page url = 'https://www.mfa.gov.sg/Where-Are-You-Travelling-To?letter={}'.format( letter, sep='') driver = create_driver() driver.get(url) #Selenium hands the page source to Beautiful Soup soup = BeautifulSoup(driver.page_source, 'lxml') #All countries for a given letter countries = soup.findAll("a", {"class": "embassy-dropdown"}) # #retrieving links for all countries of the alphabet for country in countries: country_name = country.text.lstrip().rstrip() country_iso = find_iso_of_country(country_name) if (country_iso != "" ): #Countries that don't have iso are not official counntries href = country['href'] dictionnary[country_name] = {"href": href} finally: driver.close() driver.quit() return dictionnary
def get_countries_sockets(): # Give the location of the file loc = "./e_sockets.xlsx" #mac # loc = "server/data/e_sockets.xlsx" #pc # To open Workbook wb = xlrd.open_workbook(loc) sheet = wb.sheet_by_index(0) sheet.cell_value(0, 0) arrayOfInfo = [] #loop for every row and grab the values for row in range(sheet.nrows): country_name = sheet.cell_value(row, 0) electric_potential = sheet.cell_value(row, 1) frequency = sheet.cell_value(row, 2) plug_type = sheet.cell_value(row, 3) iso = find_iso_of_country(country_name) info = { "name":country_name, "iso": iso, "plug_type": plug_type, "electric_potential": electric_potential, "frequency": frequency } arrayOfInfo.append(info) return arrayOfInfo
def save_to_MU(): LOGGER.info(f'Saving and parsing Mauritius into the databse') driver = create_driver() LOGGER.info('Begin parsing for Mauritius advisory') try: wiki_visa_url = wiki_visa_url_MU wiki_visa_ob = wiki_visa_parser(wiki_visa_url, driver) visas = wiki_visa_ob.visa_parser_table() LOGGER.success( 'Parsing for Mauritius advisory has been successfully completed') except Exception as error_msg: LOGGER.error( f'Error has occured while parsing for Mauritius advisory because of the following error: {error_msg}' ) info = {} array_info = [] # create an an sqlite_advisory object db = Database("countries.sqlite") db.drop_table("MU") db.add_table("MU", country_iso="text", name="text", advisory_text="text", visa_info="text") LOGGER.info('Saving Mauritius table into the database') try: for country in visas: iso = find_iso_of_country(country) if (iso != ""): name = country LOGGER.info(f'Saving {name}') visa = visas[country].get( 'visa') #dictionary for visa info is country{visa:text} advisory = "Not available yet" info = { "country_iso": iso, "name": name, "advisory": advisory, "visa_info": visa } array_info.append(info) print(name, " ", visa, " ", advisory) db.insert("MU", iso, name, advisory, visa) LOGGER.success( f'{name} was sucessfully saved to the database with the following information: {visa}. {advisory}.' ) LOGGER.success( 'Mauritius table successfully saved to the database') except Exception as error_msg: LOGGER.error( f'An error has occured while saving Mauritius table to the database because of the following error: {error_msg}' ) db.close_connection() quit_driver(driver) with open('./advisory-mu.json', 'w') as outfile: json.dump(array_info, outfile)
def save_to_UK(): LOGGER.info("Begin parsing and saving for United Kingdom table...") driver = create_driver() LOGGER.info('Parsing the visa requirements of all countries for United Kingdom advisory') try: wiki_visa_url ="https://en.wikipedia.org/wiki/Visa_requirements_for_British_citizens" wiki_visa_ob = wiki_visa_parser(wiki_visa_url,driver) visas = wiki_visa_ob.visa_parser_table() data = parse_all_countries_advisory() LOGGER.success('Successfully parsed the visa requirements of all countries for United Kingdom advisory') except Exception as error_msg: LOGGER.error(f'An error has occured while retrieving the visa reuirements of all countries for United Kingdom advisory because of the following error: {error_msg}') info = {} array_info = [] # create an an sqlite_advisory object] db = Database("countries.sqlite") db.drop_table("GB") db.add_table("GB", country_iso="text", name="text", advisory_text="text", visa_info="text") LOGGER.info('Saving countries informations into the UK table') try: for country in visas: iso = find_iso_of_country(country) if(iso != ""): try: name = country advisory = data[iso].get('advisory') #dictionary for the travel advisory is iso{advisory:text} visa_info = visas[country].get('visa') #dictionary for visa info is country{visa:text} info = { "country_iso" : iso, "name": name, "advisory": advisory, "visa_info": visa_info } array_info.append(info) LOGGER.success(f"Saving {name} into the UK table with the following information: {visa_info}. {advisory}") db.insert("GB",iso,name,advisory,visa_info) LOGGER.success(f'{name} sucesfully saved to the database.') except KeyError: LOGGER.warning(f'This country doesn\'t have advisory info: {country}') print("This country doesn't have advisory info: ",country) LOGGER.info(f'Its ISO is {iso}') print("Its ISO is: ",iso) LOGGER.success('All countries have been succesfully saved into the UK table') except Exception as error_msg: LOGGER.error(f'An error has occured while saving countries into the UK table because of the following: {error_msg}') db.close_connection() with open('./advisory-uk.json', 'w') as outfile: json.dump(array_info, outfile)
def get_country_traffic_side(): array_of_country_info = [] already_parsed = [] try: # this is the link to the first page url = 'https://www.worldstandards.eu/cars/list-of-left-driving-countries/' driver = create_driver() driver.get(url) # Selenium hands the page source to Beautiful Soup soup = BeautifulSoup(driver.page_source, 'html.parser') table = soup.find('table') tbody = table.find('tbody') allRows = tbody.findAll('tr') for country_row in allRows: isHeader = country_row.find( 'th') != None #The header row should be discarded if (not isHeader): country = country_row.findAll('td')[0].text traffic_side = country_row.findAll('td')[1].text country_name_has_bracket = country.find('(') if (country_name_has_bracket > -1 ): #We want to remove the bracket from the country name country = country[0:country_name_has_bracket] country_iso = find_iso_of_country(country) if (country_iso != ""): if country_iso not in already_parsed: # Only parse the main traffic side of a country if "left" in traffic_side: traffic_side = "left" else: traffic_side = "right" info = { "country_iso": country_iso, "country_name": country, "traffic_side": traffic_side } already_parsed.append(country_iso) array_of_country_info.append(info) else: print( "The main land for this country is parsed already", country) return array_of_country_info finally: driver.close() driver.quit()
def get_countries_cocainelaw(): LOGGER.info("Retrieving information for cocaine") try: # this is the link to the first page url = 'https://en.wikipedia.org/wiki/Legal_status_of_cocaine' driver = create_driver() driver.get(url) # Selenium hands the page source to Beautiful Soup soup=BeautifulSoup(driver.page_source, 'html.parser') # patter of the link to the country page that the href should match table = soup.find('table', {'class':"wikitable"}) tbody = table.find('tbody') table_rows = tbody.find_all('tr') cocaine_info= {} arrayCocaineInfo = {} for tablerow in table_rows: table_columns = tablerow.find_all('td') if(len(table_columns)>0): country_name= table_columns[0].text cocaine_possession= table_columns[1].text cocaine_possession= re.sub(r'\[\d*\]',' ',cocaine_possession.rstrip()) cocaine_sale= table_columns[2].text cocaine_sale= re.sub(r'\[\d*\]',' ',cocaine_sale.rstrip()) cocaine_transport= table_columns[3].text cocaine_transport= re.sub(r'\[\d*\]',' ',cocaine_transport.rstrip()) cocaine_cultivation= table_columns[4].text cocaine_cultivation= re.sub(r'\[\d*\]',' ',cocaine_cultivation.rstrip()) country_iso = find_iso_of_country(country_name) cocaine_info = { "name":country_name, "iso": country_iso, "cocaine-possession": cocaine_possession, "cocaine-sale": cocaine_sale, "cocaine-transport": cocaine_transport, "cocaine-cultivation": cocaine_cultivation } arrayCocaineInfo[country_iso] = cocaine_info return arrayCocaineInfo except Exception as error_msg: LOGGER.error(f'An error has occured while retrieving information for cocaine because of the following error: {error_msg}') finally: driver.close() driver.quit()
def get_countries_canabaislaw(): LOGGER.info("Retrieving information for canabais") try: # this is the link to the first page url = 'https://en.wikipedia.org/wiki/Legality_of_cannabis' driver = create_driver() driver.get(url) # Selenium hands the page source to Beautiful Soup soup=BeautifulSoup(driver.page_source, 'html.parser') # patter of the link to the country page that the href should match table = soup.find('table', {'class':"wikitable"}) tbody = table.find('tbody') table_rows = tbody.find_all('tr') canabais_info= {} arrayCanabaisInfo = {} for tablerow in table_rows: table_columns = tablerow.find_all('td') if(len(table_columns)>0): country_name= table_columns[0].text recreational= table_columns[1].text recreational= re.sub(r'\[\d*\]',' ',recreational.rstrip()) medical= table_columns[2].text medical= re.sub(r'\[\d*\]',' ',medical.rstrip()) country_iso = find_iso_of_country(country_name) canabais_info = { "name":country_name, "iso": country_iso, "canabais-recreational": recreational, "canabais-medical": medical } arrayCanabaisInfo[country_iso] = canabais_info return arrayCanabaisInfo except Exception as error_msg: LOGGER.error(f'An error has occured while retrieving information for cannabis because of the followin error: {error_msg}') finally: driver.close() driver.quit()
def get_url_of_countries(): info = {} LOGGER.info('Retrieving URL of all countries for Vaccines table') try: #this is the link to the first page driver = create_driver() driver.get(vaccine_url) #Selenium hands the page source to Beautiful Soup soup = BeautifulSoup(driver.page_source, 'lxml') #patter of the link to the country page that the href should match countries_per_letter_array = soup.find_all("ul", {"class": "list-bullet"}) for countries_per_letter in countries_per_letter_array: # print(countries_div) countries_given_letter_array = countries_per_letter.find_all('a') #retrieving links for all countries for country in countries_given_letter_array: country_name = country.text country_iso = find_iso_of_country(country_name) if ( country_iso != "" ): #Countries that don't have iso are not official counntries href = country['href'] info[country_iso] = {"href": href} LOGGER.info(f' Retrieving URL of {country_name}, {href}') except Exception as error_msg: LOGGER.error( f'Could not retrieve URLs of countries because of the following error: {error_msg}' ) finally: driver.close() driver.quit() return info
def get_additional_advisory_info_url(): url = 'https://travel.gc.ca/travelling/advisories' #set up the headless chrome driver driver = create_driver() driver.get(url) soup = BeautifulSoup(driver.page_source, 'lxml') table = soup.find('table') table_body = table.find('tbody') table_rows = table_body.find_all('tr', attrs={'class': 'gradeX'}) additional_advisory = {} for row in table_rows: cols = row.find_all('td') country = cols[0].text iso = find_iso_of_country(country) advisory = cols[2].text additional_advisory[iso] = { 'country_name': country, 'advisory_text': advisory } quit_driver(driver) return additional_advisory