def test_cisco_eox_database_query_with_server_error(self, monkeypatch): class MockSession: def get(self, *args, **kwargs): raise Exception("Server is down") monkeypatch.setattr(requests, "Session", MockSession) with pytest.raises(ConnectionFailedException) as exinfo: api_crawler.get_raw_api_data("MyQuery") assert exinfo.match("cannot contact API endpoint at")
def test_invalid_cisco_eox_database_query(self): with pytest.raises(ValueError) as exinfo: api_crawler.get_raw_api_data(None) assert exinfo.match("either year or the api_query must be provided") with pytest.raises(ValueError) as exinfo: api_crawler.get_raw_api_data(123) assert exinfo.match("api_query must be a string value")
def test_offline_invalid_update_cisco_eox_database_with_default_settings(self, monkeypatch): # mock the underlying GET request class MockSession: def get(self, *args, **kwargs): r = Response() r.status_code = 200 with open("app/ciscoeox/tests/data/cisco_eox_error_response.json") as f: r._content = f.read().encode("utf-8") return r monkeypatch.setattr(requests, "Session", MockSession) with pytest.raises(CiscoApiCallFailed) as exinfo: api_crawler.get_raw_api_data("WS-C2950G-48-EI-WS")
def test_multi_page_results(self, monkeypatch): # mock the underlying GET request class MockSession: def get(self, *args, **kwargs): global HIT_COUNT HIT_COUNT += 1 r = Response() r.status_code = 200 if HIT_COUNT == 1: with open( "app/ciscoeox/tests/data/cisco_eox_response_page_1_of_2.json" ) as f: r._content = f.read().encode("utf-8") else: with open( "app/ciscoeox/tests/data/cisco_eox_response_page_2_of_2.json" ) as f: r._content = f.read().encode("utf-8") return r monkeypatch.setattr(requests, "Session", MockSession) result = api_crawler.get_raw_api_data("WS-C2950G-48-EI-WS") assert len(result) == 3, "Three products should be imported" # every product should contain the following values for e in result: assert "EOLProductID" in e.keys() assert "EOXMigrationDetails" in e.keys() assert "ProductIDDescription" in e.keys()
def test_cisco_eox_database_query_with_server_error(self, monkeypatch): class MockSession: def get(self, *args, **kwargs): raise Exception("Server is down") monkeypatch.setattr(requests, "Session", MockSession) # verify that a test credential is set assert os.getenv( "TEST_CISCO_API_CLIENT_ID", None) is not None, "cisco api test credentials not set" assert os.getenv( "TEST_CISCO_API_CLIENT_SECRET", None) is not None, "cisco api test credentials not set" with pytest.raises(ConnectionFailedException) as exinfo: api_crawler.get_raw_api_data("MyQuery") assert exinfo.match("cannot contact API endpoint at")
def test_year_query(self, monkeypatch): # mock the underlying GET request class MockSession: def get(self, *args, **kwargs): r = Response() r.status_code = 200 with open( "app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json" ) as f: r._content = f.read().encode("utf-8") return r monkeypatch.setattr(requests, "Session", MockSession) result = api_crawler.get_raw_api_data(year=2018) assert len(result) == 3, "Three products should be imported" # every product should contain the following values for e in result: assert "EOLProductID" in e.keys() assert "EOXMigrationDetails" in e.keys() assert "ProductIDDescription" in e.keys()
def test_cisco_eox_database_query_with_disabled_cisco_eox_api(self): with pytest.raises(CiscoApiCallFailed) as exinfo: api_crawler.get_raw_api_data("MyQuery") assert exinfo.match("Cisco API access not enabled")
def initial_sync_with_cisco_eox_api(self, years_list): """ synchronize all entries from the EoX API for a given amount of years (today - n-years), ignores the create missing entries and the configurable blacklist. :param self: :param years_list: list of years to sync (e.g. [2018, 2017, 2016] :return: """ if type(years_list) is not list: raise AttributeError("years_list must be a list") for val in years_list: if type(val) is not int: raise AttributeError("years_list must be a list of integers") if len(years_list) == 0: return { "status_message": "No years provided, nothing to do." } app_config = AppSettings() # test Cisco EoX API access test_result = utils.check_cisco_eox_api_access( app_config.get_cisco_api_client_id(), app_config.get_cisco_api_client_secret(), False ) failed_years = [] successful_years = [] if test_result: # perform synchronization self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "start initial synchronization with the Cisco EoX API..." }) all_records = [] for year in years_list: self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "fetch all information for year %d..." % year }) # wait some time between the query calls time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time())) # fetch all API entries for a specific year try: records = cisco_eox_api_crawler.get_raw_api_data(year=year) successful_years += [year] all_records.append({ "year": year, "records": records }) except CiscoApiCallFailed as ex: msg = "Cisco EoX API call failed (%s)" % str(ex) logger.error("Query for year %s to Cisco EoX API failed (%s)" % (year, msg), exc_info=True) failed_years += [year] NotificationMessage.objects.create( title="Initial data import failed", summary_message="Unable to collect Cisco EoX data for year %d" % year, detailed_message=msg, type=NotificationMessage.MESSAGE_ERROR ) except Exception as ex: msg = "Unexpected Exception, cannot access the Cisco API. Please ensure that the server is " \ "connected to the internet and that the authentication settings are valid." logger.error("Query for year %s to Cisco EoX API failed (%s)" % (year, msg), exc_info=True) failed_years += [year] NotificationMessage.objects.create( title="Initial data import failed", summary_message="Unable to collect Cisco EoX data for year %d" % year, detailed_message=msg, type=NotificationMessage.MESSAGE_ERROR ) # update local database (asynchronous task) if len(all_records) != 0: tasks = [ update_local_database_records.s({}, all_records[0]["year"], all_records[0]["records"]) ] for r in all_records[1:]: tasks.append(update_local_database_records.s(r["year"], r["records"])) tasks.append(notify_initial_import_result.s()) chain(*tasks).apply_async() time.sleep(10) # remove in progress flag with the cache cache.delete("CISCO_EOX_INITIAL_SYN_IN_PROGRESS") success_msg = ",".join([str(e) for e in successful_years]) if len(success_msg) == 0: success_msg = "None" failed_msg = "" if len(failed_years) != 0: failed_msg = " (for %s the synchronization failed)" % ",".join([str(e) for e in failed_years]) return { "status_message": "The EoX data were successfully downloaded for the following years: %s%s" % (success_msg, failed_msg) }
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False): """ This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the results in the local database. There are two types of operation: * cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the blacklist and not in the database * cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already included in the database :return: """ app_config = AppSettings() run_task = app_config.is_periodic_sync_enabled() if not (run_task or ignore_periodic_sync_flag): result = { "status_message": "task not enabled" } else: logger.info("start sync with Cisco EoX API...") self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "sync with Cisco EoX API..." }) # read configuration for the Cisco EoX API synchronization queries = app_config.get_cisco_eox_api_queries_as_list() if len(queries) == 0: result = { "status_message": "No Cisco EoX API queries configured." } NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_WARNING, summary_message="There are no Cisco EoX API queries configured. Nothing to do.", detailed_message="There are no Cisco EoX API queries configured. Please configure at least on EoX API " "query in the settings or disable the periodic synchronization." ) # update the local database with the Cisco EoX API else: # test Cisco EoX API access test_result = utils.check_cisco_eox_api_access( app_config.get_cisco_api_client_id(), app_config.get_cisco_api_client_secret(), False ) if not test_result: msg = "Cannot contact Cisco EoX API, please verify your internet connection and access " \ "credentials." if not ignore_periodic_sync_flag: NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR, summary_message="The synchronization with the Cisco EoX API was not successful.", detailed_message=msg ) result = { "error_message": msg } else: # execute all queries from the configuration query_eox_records = {} failed_queries = [] failed_query_msgs = {} successful_queries = [] counter = 1 for query in queries: self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of " "%d</strong>)..." % (query, counter, len(queries)) }) # wait some time between the query calls time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time())) try: query_eox_records[query] = cisco_eox_api_crawler.get_raw_api_data(api_query=query) successful_queries.append(query) except CiscoApiCallFailed as ex: msg = "Cisco EoX API call failed (%s)" % str(ex) logger.error("Query %s to Cisco EoX API failed (%s)" % (query, msg), exc_info=True) failed_queries.append(query) failed_query_msgs[query] = str(ex) except Exception as ex: msg = "Unexpected Exception, cannot access the Cisco API. Please ensure that the server is " \ "connected to the internet and that the authentication settings are " \ "valid." logger.error("Query %s to Cisco EoX API failed (%s)" % (query, msg), exc_info=True) failed_queries.append(query) failed_query_msgs[query] = str(ex) counter += 1 for key in query_eox_records: amount_of_records = len(query_eox_records[key]) self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "update database (query <code>%s</code>, processed <b>0</b> of " "<b>%d</b> results)..." % (key, amount_of_records) }) # update database in a separate task update_cisco_eox_records.apply_async(kwargs={ "records": query_eox_records[key] }) # view the queries in the detailed message and all messages (if there are some) detailed_message = "The following queries were executed:<br><ul style=\"text-align: left;\">" for fq in failed_queries: detailed_message += "<li class=\"text-danger\"><code>%s</code> " \ "(failed, %s)</li>" % (fq, failed_query_msgs.get(fq, "unknown")) for sq in successful_queries: detailed_message += "<li><code>%s</code> (<b>affects %d products</b>, " \ "success)</li>" % (sq, len(query_eox_records[sq])) detailed_message += "</ul>" # show the executed queries in the summary message if len(failed_queries) == 0 and len(successful_queries) != 0: summary_html = "The following queries were successful executed: %s" % ", ".join( ["<code>%s</code>" % query for query in successful_queries] ) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_SUCCESS, summary_message="The synchronization with the Cisco EoX API was successful. " + summary_html, detailed_message=detailed_message ) elif len(failed_queries) != 0 and len(successful_queries) == 0: summary_html = "The following queries failed to execute: %s" % ", ".join( ["<code>%s</code>" % query for query in failed_queries] ) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR, summary_message="The synchronization with the Cisco EoX API was not successful. " + summary_html, detailed_message=detailed_message ) else: summary_html = "The following queries were successful executed: %s\n<br>The following queries " \ "failed to execute: %s" % ( ", ".join(["<code>%s</code>" % query for query in successful_queries]), ", ".join(["<code>%s</code>" % query for query in failed_queries]) ) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_WARNING, summary_message="The synchronization with the Cisco EoX API was partially " "successful. " + summary_html, detailed_message=detailed_message ) result = {"status_message": "<p style=\"text-align: left;\">" + detailed_message + "</p>"} # if the task was executed eager, set state to SUCCESS (required for testing) if self.request.is_eager: self.update_state(state=TaskState.SUCCESS, meta={"status_message": summary_html}) # remove in progress flag with the cache cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS") return result