from lib.database import Database
import pandas as pd
# Initialize flags, logger & database
FLAGS = Flags()
LEVEL = FLAGS.get_logger_level()
LOGGER = Logger(level=LEVEL) if LEVEL is not None else Logger()
DB = Database(sqlite_db)
# Create table if it does not exist
DB.add_table('un', country='text', lifeExpectancy='text', infantMortality='text', nbOfPhysicians='text', homicideRate='text', sanitation='text', water='text')
# Parse currencies and add to database
for country in iso_list:
try:
LOGGER.info(f'Beginning currency parsing for country: {country}')
url_converter = UrlHelper(united_nations_api_link)
information_link = url_converter.get_united_nations_api(country)
LOGGER.info(f'Retrieving information from following link: {information_link}')
# Scrape United Nations data endpoint with Pandas
data_tables = pd.read_html(information_link, index_col=0)
# Pick specific dataframe that will always be the third index
data_table_social = data_tables[3]
LOGGER.info(f'Parsing returned following dataframe: {data_table_social}')
# Get latest year
latest_year_social = data_table_social.columns[-1]
LOGGER.info(f'Parsing returned following year: {latest_year_social}')
email='text',
website='text')
for result in embassy_results["results"]["bindings"]:
try:
if 'city' not in result:
result['city'] = {'value': ''}
if 'operator' not in result:
result['operator'] = {'value': ''}
if 'phone' not in result:
result['phone'] = {'value': ''}
if 'email' not in result:
result['email'] = {'value': ''}
if 'website' not in result:
result['website'] = {'value': ''}
LOGGER.info(f'Getting embassy data for {result["country"]["value"]}')
DB.insert_or_update('embassies', result['country']['value'],
result['city']['value'],
result['operator']['value'],
result['type']['value'], result['phone']['value'],
result['email']['value'],
result['website']['value'])
LOGGER.success(
f'Successfully entered embassy information for {result["country"]["value"]}'
)
except Exception as error_msg:
LOGGER.error(
f'Could not get embassy data for {result["country"]["value"]} because of the following error: {error_msg}'
)
pass
LEVEL = FLAGS.get_logger_level()
LOGGER = Logger(level=LEVEL) if LEVEL is not None else Logger()
DB = Database(sqlite_db)
# Create table if it does not exist
DB.add_table('covid19',
country='text primary key',
totalcases='integer',
newcases='integer',
totaldeaths='integer',
newdeaths='integer',
totalrecovered='integer',
activecases='integer',
seriouscritical='integer')
LOGGER.info(f'Retrieving information from following link: {covid19_url}')
# Pretend to be a browser to avoid HTTP 403
header = {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36",
"X-Requested-With": "XMLHttpRequest"
}
r = get(covid19_url, headers=header)
# Scrape Worldometers data endpoint with Pandas
data_table = pd.read_html(r.text, index_col=0)
data_frame = data_table[0]
data_frame.fillna(0, inplace=True)
data_frame = data_frame.astype(str)
DB = Database(sqlite_db)
# Removing table if it was already there
DB.drop_table("emergency")
# Create table if it does not exist
DB.add_table('emergency', country='text', police='text', ambulance='text', fire='text')
data_tables = pd.read_html('http://chartsbin.com/view/1983')
data_table = data_tables[0]
latest_year = data_table.columns[1]
for country in iso_list_2:
try:
LOGGER.info(f'Getting emergency contacts data for {country}')
if str(data_table.iloc[iso_list_2.index(country)][1]) == 'nan':
police = ''
else:
police = data_table.iloc[iso_list_2.index(country)][1]
if str(data_table.iloc[iso_list_2.index(country)][2]) == 'nan':
ambulance = ''
else:
ambulance = data_table.iloc[iso_list_2.index(country)][2]
if str(data_table.iloc[iso_list_2.index(country)][3]) == 'nan':
fire = ''
else:
fire = data_table.iloc[iso_list_2.index(country)][3]
LOGGER.success(f'Following emergency contacts data was retrieved: {country}: {police} {ambulance} {fire}')
LOGGER.info('Inserting data into database.')
DB.insert_or_update('emergency', country, police, ambulance, fire)
LOGGER = Logger(level=LEVEL) if LEVEL is not None else Logger()
DB = Database(sqlite_db)
# Removing table if it was already there
DB.drop_table("currencies")
# Create table if it does not exist
DB.add_table('currencies',
country='text',
name='text',
code='text',
symbol='text')
# Parse currencies and add to database
for country in iso_list:
try:
LOGGER.info(f'Beginning currency parsing for country: {country}')
url_converter = UrlHelper(currency_api_link)
information_link = url_converter.get_currency_api(country)
LOGGER.info(
f'Retrieving information from following link: {information_link}')
currency_api = ApiHelper(url_converter.get_currency_api(country))
LOGGER.info(f'Parsing returned HTML code: {currency_api.get_code()}')
data = currency_api.get_json()['currencies'][0]
LOGGER.success(f'Following currency data was retrieved: {data}')
for k, v in data.items():
if v is None:
data[k] = "None"
