def test_fetch(self): incidents = [] incidents.append( AvalancheIncident(id=1, time=date.fromisoformat('2019-02-20'), coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769))) incidents.append( AvalancheIncident(id=2, time=date.fromisoformat('2019-02-20'), coords_utm=(109190, 6725372), coords_latlng=(60.474065, 7.882042))) dataframe_dict = XgeoFetcher().fetch(incidents) self.assertEqual(dataframe_dict[1].index[0], '2019-02-10') self.assertEqual(dataframe_dict[1].index[1], '2019-02-11') self.assertEqual(dataframe_dict[2].index[0], '2019-02-10') self.assertEqual(dataframe_dict[2].index[1], '2019-02-11') # Get the column names of the dataframe from XgeoFetcher column_names = list( map(lambda data_code: data_code[1], XgeoFetcher.DATA_CODES)) first_column_name = column_names[0] second_column_name = column_names[1] self.assertIn(first_column_name, dataframe_dict[1].columns) self.assertIn(first_column_name, dataframe_dict[2].columns) self.assertIn(second_column_name, dataframe_dict[1].columns) self.assertIn(second_column_name, dataframe_dict[2].columns)
def test_fetch(self): avalanche_incident_list = [] avalanche_incident_list.append( AvalancheIncident(id=1, time=date.fromisoformat("2019-02-20"), coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769))) avalanche_incident_list.append( AvalancheIncident(id=2, time=date.fromisoformat("2019-02-25"), coords_utm=(109190, 6725372), coords_latlng=(60.474065, 7.882042))) raw_data = SkredvarselFetcher().fetch(avalanche_incident_list) self.assertEqual(raw_data[0]["id"], 1) self.assertEqual(raw_data[1]["id"], 2) self.assertTrue(raw_data[0]["DangerLevel"])
def test_skredvarsel(self): avalanche_incident_list = [] avalanche_incident_list.append( AvalancheIncident(id=1, time=date.fromisoformat("2019-02-20"), coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769))) avalanche_incident_list.append( AvalancheIncident(id=2, time=date.fromisoformat("2019-02-25"), coords_utm=(109190, 6725372), coords_latlng=(60.474065, 7.882042))) dataframe_dict = Skredvarsel().get_data(avalanche_incident_list) database_rows = dataframe_dict["skredvarsel_data"] self.assertIn("reg_id", database_rows.columns) self.assertIn("danger_level", database_rows.columns) self.assertTrue(len(database_rows.columns) > 2)
def test_representation(self): avalanche_incident_list = [] avalanche_incident_list.append(AvalancheIncident( id=1, time=date.fromisoformat("2019-02-20"), coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769) )) converted_to_string = str(avalanche_incident_list) self.assertEqual( converted_to_string, "[AvalancheIncident with id=1, time=2019-02-20, coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769)]")
def test_fetch_data_for_skredvarsel(self): s = requests.Session() #retries = Retry(total=5, backoff_factor=1, status_forcelist=[502,503,504]) #s.mount('https://', HTTPAdapter(max_retries=retries)) avalanche_incident = AvalancheIncident( id=1, time=date.fromisoformat("2019-02-20"), coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769)) response = SkredvarselFetcher.fetch_data_from_skredvarsel( avalanche_incident, s) self.assertEqual(response[0]["ValidFrom"], "2019-02-20T00:00:00")
def test_fetch_for_code(self): incident = AvalancheIncident(id=1, time=date.fromisoformat('2019-02-20'), coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769)) code = ('rr', 'rainfall') async def fetch_with_session(): async with aiohttp.ClientSession() as s: return await XgeoFetcher().fetch_for_code(s, incident, code) loop = asyncio.get_event_loop() response = loop.run_until_complete(fetch_with_session()) response = list(response.values())[0] expected = [ 2.799999952316284, 0.699999988079071, 0.10000000149011612, 0.10000000149011612, 0, 0, 0, 0, 0, 0, 0 ] self.assertEquals(response, expected)
def test_fetch_for_incident(self): incident = AvalancheIncident(id=1, time=date.fromisoformat('2019-02-20'), coords_utm=(179633, 6782269), coords_latlng=(61.044338, 9.062769)) loop = asyncio.get_event_loop() dataframe_dict = loop.run_until_complete( XgeoFetcher().fetch_for_incident(incident)) # There should be only one value in the returned response dataframe = list(dataframe_dict.values())[0] # Get the column names of the dataframe from XgeoFetcher column_names = list( map(lambda data_code: data_code[1], XgeoFetcher.DATA_CODES)) first_column_name = column_names[0] self.assertEqual(dataframe.index[0], '2019-02-10') self.assertIn(first_column_name, dataframe.columns) self.assertTrue(len(dataframe[first_column_name]) != 0)
def test_fetch_observations_for_incident(self): incidents = (AvalancheIncident(342, dt.datetime(2020, 5, 3), (0, 0), (59, 10))) async def helper(): async with aiohttp.ClientSession( auth=FrostFetcher._FrostFetcher__frost_auth) as s: return await self.fetcher.fetch_observations( s, incidents, 'SN30000', '2020-05-01', '2020-05-04', 'sum(precipitation_amount P1D)', 7.4176453769) loop = asyncio.get_event_loop() actual = loop.run_until_complete(helper()) expected = pd.DataFrame( [[ 'SN30000', 'sum(precipitation_amount P1D)', dt.datetime(2020, 5, 1), 342, 7.4176453769, 19, None, 'mm', None, None, None, None, 'PT6H', 'P1D', 0, 'C', '2', 0, None, None ], [ 'SN30000', 'sum(precipitation_amount P1D)', dt.datetime(2020, 5, 2), 342, 7.4176453769, 0, None, 'mm', None, None, None, None, 'PT6H', 'P1D', 0, 'C', '2', 0, None, None ], [ 'SN30000', 'sum(precipitation_amount P1D)', dt.datetime(2020, 5, 3), 342, 7.4176453769, 0, None, 'mm', None, None, None, None, 'PT6H', 'P1D', 0, 'C', '2', 0, None, None ]], columns=FrostFetcher.observations_headers) print('Observations actual length:', len(actual)) print('Observations expected length:', len(expected)) print(actual) print(expected) self.assertEqual(actual, expected)
def test_fetch_single_incident(self): incidents = (AvalancheIncident(342, dt.datetime(2020, 5, 3), (0, 0), (59, 10)), ) fetcher = FrostFetcher() result = fetcher.fetch(incidents) sources = result['frost_sources'] observations = result['frost_observations'] # Sort and trim the dataframes for easier testing sources = sources.head(3).reset_index(drop=True) observations = observations.sort_values( ['element', 'source', 'time', 'value', 'time_offset'], ascending=[False, True, True, False, False]).head(3).reset_index(drop=True) expected_sources = pd.DataFrame( [ [ 'SN30000', 'SensorSystem', 'LARVIK', 'Larvik', 'Norge', 'NO', None, 59.0572, 10.0667, 28, # See comment in frost_initializer.py # dt.datetime.fromisoformat('1883-06-01T00:00:00.000'), # None, 'VESTFOLD OG TELEMARK', 38, 'LARVIK', 3805, "['LARVIK KOMMUNE']", None, None, None, '0-578-0-30000' ], [ 'SN27780', 'SensorSystem', 'TJØLLING', 'Tjølling', 'Norge', 'NO', None, 59.0467, 10.125, 19, # See comment in frost_initializer.py # dt.datetime.fromisoformat('2005-01-01T00:00:00.000'), # None, 'VESTFOLD OG TELEMARK', 38, 'LARVIK', 3805, "['NIBIO']", "['1']", None, None, '0-578-0-27780' ], [ 'SN30249', 'SensorSystem', 'LANGANGEN', # God damn stray space 'Langangen ', 'Norge', 'NO', None, 59.0875, 9.802, 5, # See comment in frost_initializer.py # dt.datetime.fromisoformat('2015-03-18T00:00:00.000'), # None, 'VESTFOLD OG TELEMARK', 38, 'PORSGRUNN', 3806, "['PORSGRUNN KOMMUNE']", "['501151102']", None, None, '0-578-0-30249' ] ], columns=FrostFetcher.sources_headers).reset_index(drop=True) expected_observations = pd.DataFrame( [[ 'SN27630', 'sum(precipitation_amount P1D)', dt.datetime.fromisoformat('2020-05-01T00:00:00.000'), 342, 15.54413153464, 20.4, None, 'mm', None, None, None, None, 'PT6H', 'P1D', 0, 'C', '2', 0, None, None ], [ 'SN27630', 'sum(precipitation_amount P1D)', dt.datetime.fromisoformat('2020-05-01T00:00:00.000'), 342, 15.54413153464, 16.9, None, 'mm', None, None, None, None, 'PT18H', 'P1D', 0, 'C', '2', 2, None, None ], [ 'SN27630', 'sum(precipitation_amount P1D)', dt.datetime.fromisoformat('2020-05-02T00:00:00.000'), 342, 15.54413153464, 0.0, None, 'mm', None, None, None, None, 'PT6H', 'P1D', 0, 'C', '2', 0, None, None ]], columns=FrostFetcher.observations_headers).reset_index(drop=True) self.assertEqual(sources, expected_sources) self.assertEqual(observations, expected_observations)
def main(): # See configuration.ini for details fetch_regobs, load_excel_data, api_fetch_list, api_delete_list, api_initialize_list = load_configuration() # Handle command line arguments force_update = parse_command_line_arguments() logging.info( 'Application started with force_update={}'.format(force_update)) # Create engine and db_inserter try: engine = create_db_connection() except Exception as e: logging.exception('Cannot connect to the database') raise e db_manager = DbManager(engine) if load_excel_data: logging.info('Adding excel data to database table excel_data..') excel_data = process_excel_data() ExcelData.metadata.create_all(engine) db_manager.insert_dataframe('excel_data', excel_data, if_exists='replace') logging.info('Fetching RegObs data..') # Fetch regobs data from api if fetch_regobs: try: api_data = Regobs().get_data() except Exception as e: logging.exception('Cannot fetch RegObs data') raise e to_csv(api_data, 'csv_files/regobs.csv') # Load regobs data from csv file (can be useful for debugging or testing incremental update) else: api_data = read_csv('csv_files/regobs.csv') # Incremental update. Only update added, updated or deleted records in database tables. if not force_update: # Specify that the dataframe should be appended to the existing data in the database tables if_table_exists_in_database = 'append' # Query current data in database logging.info('Querying regobs table from database..') try: db_data = db_manager.query_all_data_from_table( 'regobs_data', 'reg_id') except exc.NoSuchTableError as e: logging.exception( 'The table regobs_data does not exist in the database. Run the application with --force-update command line parameter to initialize all tables and fetch all data.') raise e except Exception as e: logging.exception('Cannot query RegObs data from database') raise e # Compare current database data with new api data logging.info( 'Comparing dataframes to determine which rows are added or removed..') # Rows to delete from all tables deleted_rows = dataframe_difference( db_data, api_data, ['reg_id', 'dt_change_time']) # Rows to add to all tables new_rows = dataframe_difference( api_data, db_data, ['reg_id', 'dt_change_time']) deleted_reg_ids = list(deleted_rows['reg_id']) deleted_reg_ids = [int(x) for x in deleted_reg_ids] logging.info('Records with the following reg_ids will be deleted from the database: {}'.format( deleted_reg_ids)) if deleted_reg_ids: # Delete removed rows from api's try: for data_class in api_delete_list: logging.info( 'Deleting removed records for: {}'.format(data_class.__name__)) db_manager.delete_rows_with_reg_id( deleted_reg_ids, data_class) except Exception as e: logging.exception( 'Cannot delete removed records from database table') raise e else: logging.info( 'There are no deleted records to remove from the database') if not new_rows.empty: logging.info( 'Number of new records to add: {}'.format(len(new_rows))) try: avalanche_incident_list = AvalancheIncident.from_dataframe( new_rows ) except Exception as e: logging.exception( 'Cannot create avalanche_incident_list from regobs data') raise e # Append new rows to regobs table try: insert_regobs_data_to_database(new_rows, db_manager, 'append') except Exception as e: logging.exception( 'Cannot append RegObs data to database table') raise e else: avalanche_incident_list = [] # Initialize database and load all data elif force_update: # Specify that the dataframe should replace existing data in the database table if_table_exists_in_database = 'replace' try: avalanche_incident_list = AvalancheIncident.from_dataframe( api_data ) except Exception as e: logging.exception( 'Cannot create avalanche_incident_list from regobs data') raise e logging.info('Initializing database tables..') try: initialize_tables(api_initialize_list, engine) except Exception as e: logging.exception( 'Cannot initialize tables in database') raise e try: insert_regobs_data_to_database(api_data, db_manager, 'replace') except Exception as e: logging.exception( 'Cannot add RegObs data to database table') raise e if not avalanche_incident_list: logging.info('There is no new records to add to the database') logging.info('The application terminated successfully') return try: api_table_dict = get_table_dict_for_apis_in_list( api_fetch_list, avalanche_incident_list) except Exception as e: logging.exception( 'Error fetching API data') raise e # Set new database connection db_manager.engine = create_db_connection() try: insert_data_for_table_dict( api_table_dict, db_manager, if_table_exists_in_database) except Exception as e: logging.exception( 'Cannot add API data to database table') raise e logging.info('The application terminated successfully')