def test_fetch(self):
incidents = []
incidents.append(
AvalancheIncident(id=1,
time=date.fromisoformat('2019-02-20'),
coords_utm=(179633, 6782269),
coords_latlng=(61.044338, 9.062769)))
incidents.append(
AvalancheIncident(id=2,
time=date.fromisoformat('2019-02-20'),
coords_utm=(109190, 6725372),
coords_latlng=(60.474065, 7.882042)))
dataframe_dict = XgeoFetcher().fetch(incidents)
self.assertEqual(dataframe_dict[1].index[0], '2019-02-10')
self.assertEqual(dataframe_dict[1].index[1], '2019-02-11')
self.assertEqual(dataframe_dict[2].index[0], '2019-02-10')
self.assertEqual(dataframe_dict[2].index[1], '2019-02-11')
# Get the column names of the dataframe from XgeoFetcher
column_names = list(
map(lambda data_code: data_code[1], XgeoFetcher.DATA_CODES))
first_column_name = column_names[0]
second_column_name = column_names[1]
self.assertIn(first_column_name, dataframe_dict[1].columns)
self.assertIn(first_column_name, dataframe_dict[2].columns)
self.assertIn(second_column_name, dataframe_dict[1].columns)
self.assertIn(second_column_name, dataframe_dict[2].columns)
def test_fetch(self):
avalanche_incident_list = []
avalanche_incident_list.append(
AvalancheIncident(id=1,
time=date.fromisoformat("2019-02-20"),
coords_utm=(179633, 6782269),
coords_latlng=(61.044338, 9.062769)))
avalanche_incident_list.append(
AvalancheIncident(id=2,
time=date.fromisoformat("2019-02-25"),
coords_utm=(109190, 6725372),
coords_latlng=(60.474065, 7.882042)))
raw_data = SkredvarselFetcher().fetch(avalanche_incident_list)
self.assertEqual(raw_data[0]["id"], 1)
self.assertEqual(raw_data[1]["id"], 2)
self.assertTrue(raw_data[0]["DangerLevel"])
def test_skredvarsel(self):
avalanche_incident_list = []
avalanche_incident_list.append(
AvalancheIncident(id=1,
time=date.fromisoformat("2019-02-20"),
coords_utm=(179633, 6782269),
coords_latlng=(61.044338, 9.062769)))
avalanche_incident_list.append(
AvalancheIncident(id=2,
time=date.fromisoformat("2019-02-25"),
coords_utm=(109190, 6725372),
coords_latlng=(60.474065, 7.882042)))
dataframe_dict = Skredvarsel().get_data(avalanche_incident_list)
database_rows = dataframe_dict["skredvarsel_data"]
self.assertIn("reg_id", database_rows.columns)
self.assertIn("danger_level", database_rows.columns)
self.assertTrue(len(database_rows.columns) > 2)
def test_representation(self):
avalanche_incident_list = []
avalanche_incident_list.append(AvalancheIncident(
id=1,
time=date.fromisoformat("2019-02-20"),
coords_utm=(179633, 6782269),
coords_latlng=(61.044338, 9.062769)
))
converted_to_string = str(avalanche_incident_list)
self.assertEqual(
converted_to_string, "[AvalancheIncident with id=1, time=2019-02-20, coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769)]")
def test_fetch_data_for_skredvarsel(self):
s = requests.Session()
#retries = Retry(total=5, backoff_factor=1, status_forcelist=[502,503,504])
#s.mount('https://', HTTPAdapter(max_retries=retries))
avalanche_incident = AvalancheIncident(
id=1,
time=date.fromisoformat("2019-02-20"),
coords_utm=(179633, 6782269),
coords_latlng=(61.044338, 9.062769))
response = SkredvarselFetcher.fetch_data_from_skredvarsel(
avalanche_incident, s)
self.assertEqual(response[0]["ValidFrom"], "2019-02-20T00:00:00")
def test_fetch_for_code(self):
incident = AvalancheIncident(id=1,
time=date.fromisoformat('2019-02-20'),
coords_utm=(179633, 6782269),
coords_latlng=(61.044338, 9.062769))
code = ('rr', 'rainfall')
async def fetch_with_session():
async with aiohttp.ClientSession() as s:
return await XgeoFetcher().fetch_for_code(s, incident, code)
loop = asyncio.get_event_loop()
response = loop.run_until_complete(fetch_with_session())
response = list(response.values())[0]
expected = [
2.799999952316284, 0.699999988079071, 0.10000000149011612,
0.10000000149011612, 0, 0, 0, 0, 0, 0, 0
]
self.assertEquals(response, expected)
def test_fetch_for_incident(self):
incident = AvalancheIncident(id=1,
time=date.fromisoformat('2019-02-20'),
coords_utm=(179633, 6782269),
coords_latlng=(61.044338, 9.062769))
loop = asyncio.get_event_loop()
dataframe_dict = loop.run_until_complete(
XgeoFetcher().fetch_for_incident(incident))
# There should be only one value in the returned response
dataframe = list(dataframe_dict.values())[0]
# Get the column names of the dataframe from XgeoFetcher
column_names = list(
map(lambda data_code: data_code[1], XgeoFetcher.DATA_CODES))
first_column_name = column_names[0]
self.assertEqual(dataframe.index[0], '2019-02-10')
self.assertIn(first_column_name, dataframe.columns)
self.assertTrue(len(dataframe[first_column_name]) != 0)
def test_fetch_observations_for_incident(self):
incidents = (AvalancheIncident(342, dt.datetime(2020, 5, 3), (0, 0),
(59, 10)))
async def helper():
async with aiohttp.ClientSession(
auth=FrostFetcher._FrostFetcher__frost_auth) as s:
return await self.fetcher.fetch_observations(
s, incidents, 'SN30000', '2020-05-01', '2020-05-04',
'sum(precipitation_amount P1D)', 7.4176453769)
loop = asyncio.get_event_loop()
actual = loop.run_until_complete(helper())
expected = pd.DataFrame(
[[
'SN30000', 'sum(precipitation_amount P1D)',
dt.datetime(2020, 5,
1), 342, 7.4176453769, 19, None, 'mm', None, None,
None, None, 'PT6H', 'P1D', 0, 'C', '2', 0, None, None
],
[
'SN30000', 'sum(precipitation_amount P1D)',
dt.datetime(2020, 5,
2), 342, 7.4176453769, 0, None, 'mm', None, None,
None, None, 'PT6H', 'P1D', 0, 'C', '2', 0, None, None
],
[
'SN30000', 'sum(precipitation_amount P1D)',
dt.datetime(2020, 5,
3), 342, 7.4176453769, 0, None, 'mm', None, None,
None, None, 'PT6H', 'P1D', 0, 'C', '2', 0, None, None
]],
columns=FrostFetcher.observations_headers)
print('Observations actual length:', len(actual))
print('Observations expected length:', len(expected))
print(actual)
print(expected)
self.assertEqual(actual, expected)
def test_fetch_single_incident(self):
incidents = (AvalancheIncident(342, dt.datetime(2020, 5, 3), (0, 0),
(59, 10)), )
fetcher = FrostFetcher()
result = fetcher.fetch(incidents)
sources = result['frost_sources']
observations = result['frost_observations']
# Sort and trim the dataframes for easier testing
sources = sources.head(3).reset_index(drop=True)
observations = observations.sort_values(
['element', 'source', 'time', 'value', 'time_offset'],
ascending=[False, True, True, False,
False]).head(3).reset_index(drop=True)
expected_sources = pd.DataFrame(
[
[
'SN30000',
'SensorSystem',
'LARVIK',
'Larvik',
'Norge',
'NO',
None,
59.0572,
10.0667,
28,
# See comment in frost_initializer.py
# dt.datetime.fromisoformat('1883-06-01T00:00:00.000'),
# None,
'VESTFOLD OG TELEMARK',
38,
'LARVIK',
3805,
"['LARVIK KOMMUNE']",
None,
None,
None,
'0-578-0-30000'
],
[
'SN27780',
'SensorSystem',
'TJØLLING',
'Tjølling',
'Norge',
'NO',
None,
59.0467,
10.125,
19,
# See comment in frost_initializer.py
# dt.datetime.fromisoformat('2005-01-01T00:00:00.000'),
# None,
'VESTFOLD OG TELEMARK',
38,
'LARVIK',
3805,
"['NIBIO']",
"['1']",
None,
None,
'0-578-0-27780'
],
[
'SN30249',
'SensorSystem',
'LANGANGEN',
# God damn stray space
'Langangen ',
'Norge',
'NO',
None,
59.0875,
9.802,
5,
# See comment in frost_initializer.py
# dt.datetime.fromisoformat('2015-03-18T00:00:00.000'),
# None,
'VESTFOLD OG TELEMARK',
38,
'PORSGRUNN',
3806,
"['PORSGRUNN KOMMUNE']",
"['501151102']",
None,
None,
'0-578-0-30249'
]
],
columns=FrostFetcher.sources_headers).reset_index(drop=True)
expected_observations = pd.DataFrame(
[[
'SN27630', 'sum(precipitation_amount P1D)',
dt.datetime.fromisoformat('2020-05-01T00:00:00.000'), 342,
15.54413153464, 20.4, None, 'mm', None, None, None, None,
'PT6H', 'P1D', 0, 'C', '2', 0, None, None
],
[
'SN27630', 'sum(precipitation_amount P1D)',
dt.datetime.fromisoformat('2020-05-01T00:00:00.000'), 342,
15.54413153464, 16.9, None, 'mm', None, None, None, None,
'PT18H', 'P1D', 0, 'C', '2', 2, None, None
],
[
'SN27630', 'sum(precipitation_amount P1D)',
dt.datetime.fromisoformat('2020-05-02T00:00:00.000'), 342,
15.54413153464, 0.0, None, 'mm', None, None, None, None,
'PT6H', 'P1D', 0, 'C', '2', 0, None, None
]],
columns=FrostFetcher.observations_headers).reset_index(drop=True)
self.assertEqual(sources, expected_sources)
self.assertEqual(observations, expected_observations)
def main():
# See configuration.ini for details
fetch_regobs, load_excel_data, api_fetch_list, api_delete_list, api_initialize_list = load_configuration()
# Handle command line arguments
force_update = parse_command_line_arguments()
logging.info(
'Application started with force_update={}'.format(force_update))
# Create engine and db_inserter
try:
engine = create_db_connection()
except Exception as e:
logging.exception('Cannot connect to the database')
raise e
db_manager = DbManager(engine)
if load_excel_data:
logging.info('Adding excel data to database table excel_data..')
excel_data = process_excel_data()
ExcelData.metadata.create_all(engine)
db_manager.insert_dataframe('excel_data', excel_data, if_exists='replace')
logging.info('Fetching RegObs data..')
# Fetch regobs data from api
if fetch_regobs:
try:
api_data = Regobs().get_data()
except Exception as e:
logging.exception('Cannot fetch RegObs data')
raise e
to_csv(api_data, 'csv_files/regobs.csv')
# Load regobs data from csv file (can be useful for debugging or testing incremental update)
else:
api_data = read_csv('csv_files/regobs.csv')
# Incremental update. Only update added, updated or deleted records in database tables.
if not force_update:
# Specify that the dataframe should be appended to the existing data in the database tables
if_table_exists_in_database = 'append'
# Query current data in database
logging.info('Querying regobs table from database..')
try:
db_data = db_manager.query_all_data_from_table(
'regobs_data', 'reg_id')
except exc.NoSuchTableError as e:
logging.exception(
'The table regobs_data does not exist in the database. Run the application with --force-update command line parameter to initialize all tables and fetch all data.')
raise e
except Exception as e:
logging.exception('Cannot query RegObs data from database')
raise e
# Compare current database data with new api data
logging.info(
'Comparing dataframes to determine which rows are added or removed..')
# Rows to delete from all tables
deleted_rows = dataframe_difference(
db_data, api_data, ['reg_id', 'dt_change_time'])
# Rows to add to all tables
new_rows = dataframe_difference(
api_data, db_data, ['reg_id', 'dt_change_time'])
deleted_reg_ids = list(deleted_rows['reg_id'])
deleted_reg_ids = [int(x) for x in deleted_reg_ids]
logging.info('Records with the following reg_ids will be deleted from the database: {}'.format(
deleted_reg_ids))
if deleted_reg_ids:
# Delete removed rows from api's
try:
for data_class in api_delete_list:
logging.info(
'Deleting removed records for: {}'.format(data_class.__name__))
db_manager.delete_rows_with_reg_id(
deleted_reg_ids, data_class)
except Exception as e:
logging.exception(
'Cannot delete removed records from database table')
raise e
else:
logging.info(
'There are no deleted records to remove from the database')
if not new_rows.empty:
logging.info(
'Number of new records to add: {}'.format(len(new_rows)))
try:
avalanche_incident_list = AvalancheIncident.from_dataframe(
new_rows
)
except Exception as e:
logging.exception(
'Cannot create avalanche_incident_list from regobs data')
raise e
# Append new rows to regobs table
try:
insert_regobs_data_to_database(new_rows, db_manager, 'append')
except Exception as e:
logging.exception(
'Cannot append RegObs data to database table')
raise e
else:
avalanche_incident_list = []
# Initialize database and load all data
elif force_update:
# Specify that the dataframe should replace existing data in the database table
if_table_exists_in_database = 'replace'
try:
avalanche_incident_list = AvalancheIncident.from_dataframe(
api_data
)
except Exception as e:
logging.exception(
'Cannot create avalanche_incident_list from regobs data')
raise e
logging.info('Initializing database tables..')
try:
initialize_tables(api_initialize_list, engine)
except Exception as e:
logging.exception(
'Cannot initialize tables in database')
raise e
try:
insert_regobs_data_to_database(api_data, db_manager, 'replace')
except Exception as e:
logging.exception(
'Cannot add RegObs data to database table')
raise e
if not avalanche_incident_list:
logging.info('There is no new records to add to the database')
logging.info('The application terminated successfully')
return
try:
api_table_dict = get_table_dict_for_apis_in_list(
api_fetch_list, avalanche_incident_list)
except Exception as e:
logging.exception(
'Error fetching API data')
raise e
# Set new database connection
db_manager.engine = create_db_connection()
try:
insert_data_for_table_dict(
api_table_dict, db_manager, if_table_exists_in_database)
except Exception as e:
logging.exception(
'Cannot add API data to database table')
raise e
logging.info('The application terminated successfully')