def test_cisco_eox_database_query_with_server_error(self, monkeypatch): def raise_error(): raise Exception("Server is down") monkeypatch.setattr(requests, "get", raise_error) with pytest.raises(ConnectionFailedException) as exinfo: api_crawler.update_cisco_eox_database("MyQuery") assert exinfo.match("cannot contact API endpoint at")
def test_offline_invalid_update_cisco_eox_database_with_default_settings(self, monkeypatch): # mock the underlying GET request def mock_response(): r = Response() r.status_code = 200 with open("app/ciscoeox/tests/data/cisco_eox_error_response.json") as f: r._content = f.read().encode("utf-8") return r monkeypatch.setattr(requests, "get", lambda x, headers: mock_response()) with pytest.raises(CiscoApiCallFailed) as exinfo: api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS") assert exinfo.match("Cisco EoX API error: Some unknown error occurred during the API access \(YXCABC\)")
def test_offline_valid_update_cisco_eox_database_with_multiple_urls_in_api_reponse(self, monkeypatch): # mock the underlying GET request def mock_response(): r = Response() r.status_code = 200 with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f: content = f.read() # modify the URL parameter in the migration data (every import should fail) json_content = json.loads(content) values = [ "http://localhost;", "http://localhost;", "http://localhost and http://another-localhost", ] for c in range(0, len(json_content["EOXRecord"])): json_content["EOXRecord"][c]["EOXMigrationDetails"]["MigrationProductInfoURL"] = values[c % 3] r._content = json.dumps(json_content).encode("utf-8") return r monkeypatch.setattr(requests, "get", lambda x, headers: mock_response()) result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS") assert len(result) == 3, "Three products should be detected" # every body should contain a specific message for e in result: assert "message" in e assert e["message"] == "multiple URL values received, only the first one is saved" assert Product.objects.count() == 3, "Products are created" assert ProductMigrationOption.objects.count() == 3, "Migration data are created"
def test_offline_no_result_update_cisco_eox_database_with_create_flag(self, monkeypatch): # mock the underlying GET request def mock_response(): r = Response() r.status_code = 200 with open("app/ciscoeox/tests/data/cisco_eox_no_result_response.json") as f: r._content = f.read().encode("utf-8") return r monkeypatch.setattr(requests, "get", lambda x, headers: mock_response()) result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS") assert len(result) == 1, "Three products should be imported" assert "blacklist" in result[0] assert "updated" in result[0] assert "created" in result[0] assert "PID" in result[0] assert "message" in result[0] assert result[0]["blacklist"] is False assert result[0]["updated"] is False assert result[0]["created"] is False assert result[0]["PID"] is None assert result[0]["message"] == "No product update required" assert Product.objects.count() == 0, "No products are created, because the creation mode is disabled by default"
def test_offline_valid_update_cisco_eox_database_with_multiple_urls_in_result(self, monkeypatch): # mock the underlying GET request def mock_response(): r = Response() r.status_code = 200 with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f: raw_data = f.read() jdata = json.loads(raw_data) jdata["EOXRecord"][0]["EOLProductID"] = "TEST_PID" jdata["EOXRecord"][0]["LinkToProductBulletinURL"] = "http://somewhere.com/index.html ," \ "https://other.com/index.html" r._content = json.dumps(jdata).encode("utf-8") return r monkeypatch.setattr(requests, "get", lambda x, headers: mock_response()) result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS") assert len(result) == 3, "Three products should be seen in the API response" assert Product.objects.count() == 3, "Three products should be imported to the database" p = Product.objects.get(product_id="TEST_PID") assert p.eol_reference_url == "http://somewhere.com/index.html", "only the first entry is stored in the " \ "database"
def test_offline_valid_update_cisco_eox_database_with_default_settings(self, monkeypatch): # mock the underlying GET request def mock_response(): r = Response() r.status_code = 200 with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f: r._content = f.read().encode("utf-8") return r monkeypatch.setattr(requests, "get", lambda x, headers: mock_response()) result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS") assert len(result) == 3, "Three products should be imported" # every product should contain the following values for e in result: assert "blacklist" in e assert "updated" in e assert "created" in e assert "PID" in e assert "message" in e assert e["blacklist"] is False assert e["updated"] is False assert e["created"] is False assert type(e["PID"]) == str assert e["message"] is None assert Product.objects.count() == 0, "No products are created, because the creation mode is disabled by default"
def test_offline_valid_update_cisco_eox_database_with_create_flag(self, monkeypatch): # mock the underlying GET request def mock_response(): r = Response() r.status_code = 200 with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f: r._content = f.read().encode("utf-8") return r monkeypatch.setattr(requests, "get", lambda x, headers: mock_response()) result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS") assert len(result) == 3, "Three products should be imported" # every product should contain the following values for e in result: assert "blacklist" in e assert "updated" in e assert "created" in e assert "PID" in e assert "message" in e assert e["blacklist"] is False assert e["updated"] is True assert e["created"] is True assert type(e["PID"]) == str assert e["message"] is None assert Product.objects.count() == 3, "Products from the test API results are created" # call the same query again result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS") for e in result: assert "blacklist" in e assert "updated" in e assert "created" in e assert "PID" in e assert "message" in e assert e["blacklist"] is False assert e["updated"] is False assert e["created"] is False assert type(e["PID"]) == str assert e["message"] == "update suppressed (data not modified)" assert len(result) == 3, "Nothing should change"
def cisco_eox_query(request): """Manual query page against the Cisco EoX Version 5 API (if enabled) :param request: :return: """ app_config = AppSettings() app_config.read_file() cisco_api_enabled = app_config.is_cisco_api_enabled() context = {"is_cisco_api_enabled": cisco_api_enabled} if request.method == "POST": # create a form instance and populate it with data from the request: if "sync_cisco_eox_states_now" in request.POST.keys(): if "sync_cisco_eox_states_query" in request.POST.keys(): query = request.POST['sync_cisco_eox_states_query'] if query != "": if len(query.split(" ")) == 1: context['query_executed'] = query try: eox_api_update_records = update_cisco_eox_database( api_query=query) except ConnectionFailedException as ex: eox_api_update_records = [ "Cannot contact Cisco API, error message:\n%s" % ex ] except CiscoApiCallFailed as ex: eox_api_update_records = [ex] except Exception as ex: logger.debug( "execution failed due to unexpected exception", exc_info=True) eox_api_update_records = [ "execution failed: %s" % ex ] context['eox_api_update_records'] = json.dumps( eox_api_update_records, indent=4, sort_keys=True) else: context['eox_api_update_records'] = "Invalid query '%s': not executed" % \ request.POST['sync_cisco_eox_states_query'] else: context['eox_api_update_records'] = [ "Please specify a valid query" ] else: context[ 'eox_api_update_records'] = "Query not executed, please select the \"execute it now\" checkbox." return render(request, "ciscoeox/cisco_eox_query.html", context=context)
def cisco_eox_query(request): """Manual query page against the Cisco EoX Version 5 API (if enabled) :param request: :return: """ app_config = AppSettings() cisco_api_enabled = app_config.is_cisco_api_enabled() context = { "is_cisco_api_enabled": cisco_api_enabled } if request.method == "POST": # create a form instance and populate it with data from the request: if "sync_cisco_eox_states_now" in request.POST.keys(): if "sync_cisco_eox_states_query" in request.POST.keys(): query = request.POST['sync_cisco_eox_states_query'] if query != "": if len(query.split(" ")) == 1: context['query_executed'] = query try: eox_api_update_records = api_crawler.update_cisco_eox_database(api_query=query) except ConnectionFailedException as ex: eox_api_update_records = {"error": "Cannot contact Cisco API, error message:\n%s" % ex} except CiscoApiCallFailed as ex: eox_api_update_records = {"error": "Cisco API call failed: %s" % ex} except Exception as ex: # catch any exception logger.debug("execution failed due to unexpected exception", exc_info=True) eox_api_update_records = ["execution failed: %s" % ex] context['eox_api_update_records'] = json.dumps(eox_api_update_records, indent=4, sort_keys=True) else: context['eox_api_update_records'] = "Invalid query '%s': not executed" % \ request.POST['sync_cisco_eox_states_query'] else: context['eox_api_update_records'] = ["Please specify a valid query"] else: context['eox_api_update_records'] = "Query not specified." else: context['eox_api_update_records'] = "Query not executed, please select the \"execute it now\" checkbox." return render(request, "ciscoeox/cisco_eox_query.html", context=context)
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False): """ This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the results in the local database. There are two types of operation: * cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the blacklist and not in the database * cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already included in the database :return: """ app_config = AppSettings() app_config.read_file() run_task = app_config.is_cisco_eox_api_auto_sync_enabled() if ignore_periodic_sync_flag: run_task = True if run_task: logger.info("start sync with Cisco EoX API...") self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "sync with Cisco EoX API..." }) # read queries from configuration queries = app_config.get_cisco_eox_api_queries().splitlines() if len(queries) == 0: result = { "status_message": "No Cisco EoX API queries configured." } # update the local database with the Cisco EoX API else: # test Cisco EoX API access success, _ = test_cisco_eox_api_access( app_config.get_cisco_api_client_id(), app_config.get_cisco_api_client_secret(), False ) if not success: msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \ "and that the authentication settings are valid." logger.error(msg, exc_info=True) NotificationMessage.objects.create( title="Synchronization with Cisco EoX API", type=NotificationMessage.MESSAGE_ERROR, summary_message=msg, detailed_message="The synchronization with the Cisco EoX API was not started." ) result = { "error_message": msg } else: try: # execute query by query notify_metrics = { "queries": {} } counter = 0 for query in queries: self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of " "%d</strong>)..." % (query, counter + 1, len(queries)) }) query_results = cisco_eox_api_crawler.update_cisco_eox_database(query) blist_counter = 0 update_counter = 0 create_counter = 0 for qres in query_results: if qres["created"]: create_counter += 1 elif qres["updated"]: update_counter += 1 elif qres["blacklist"]: blist_counter += 1 notify_metrics["queries"][query] = { "amount": len(query_results), "updated_entries": update_counter, "created_entries": create_counter, "blacklisted_entries": blist_counter, "result_details": query_results } counter += 1 # create NotificationMessage detailed_html = "" blist_counter = 0 update_counter = 0 create_counter = 0 for query_key in notify_metrics["queries"].keys(): update_counter += notify_metrics["queries"][query_key]["updated_entries"] create_counter += notify_metrics["queries"][query_key]["created_entries"] blist_counter += notify_metrics["queries"][query_key]["blacklisted_entries"] # build detailed string detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key cond_1 = notify_metrics["queries"][query_key]["updated_entries"] == 0 cond_1 = cond_1 and (notify_metrics["queries"][query_key]["created_entries"] == 0) cond_1 = cond_1 and (notify_metrics["queries"][query_key]["blacklisted_entries"] == 0) if cond_1: detailed_html += "No changes required." else: detailed_html += "The following products are affected by this update:</p>" detailed_html += "<ul>" for qres in notify_metrics["queries"][query_key]["result_details"]: msg = "" if "message" in qres.keys(): if qres["message"]: msg = qres["message"] if qres["created"]: detailed_html += "<li>create the Product <code>%s</code> in the database" % ( qres["PID"] ) if msg != "": detailed_html += "(%s)</li>" % msg else: detailed_html += "</li>" elif qres["updated"]: detailed_html += "<li>update the Product data for <code>%s</code></li>" % ( qres["PID"] ) if msg != "": detailed_html += "(%s)</li>" % msg else: detailed_html += "</li>" elif qres["blacklist"]: detailed_html += "<li>Product data for <code>%s</code> ignored</li>" % ( qres["PID"] ) if msg != "": detailed_html += "(%s)</li>" % msg else: detailed_html += "</li>" detailed_html += "</ul>" detailed_html += "</div>" summary_html = "The synchronization was performed successfully. " if update_counter == 1: summary_html += "<strong>%d</strong> product was updated, " % update_counter else: summary_html += "<strong>%d</strong> products are updated, " % update_counter if create_counter == 1: summary_html += "<strong>%s</strong> product was added to the database and " % create_counter else: summary_html += "<strong>%s</strong> products are added to the database and " % create_counter if blist_counter == 1: summary_html += "<strong>%d</strong> product was ignored." % blist_counter else: summary_html += "<strong>%d</strong> products are ignored." % blist_counter NotificationMessage.objects.create( title="Synchronization with Cisco EoX API", type=NotificationMessage.MESSAGE_SUCCESS, summary_message=summary_html, detailed_message=detailed_html ) result = { "status_message": detailed_html } # if the task was executed eager, set state to SUCCESS (required for testing) if self.request.is_eager: self.update_state(state=TaskState.SUCCESS, meta={ "status_message": detailed_html }) except CredentialsNotFoundException as ex: msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(ex) logger.error(msg, exc_info=True) NotificationMessage.objects.create( title="Synchronization with Cisco EoX API", type=NotificationMessage.MESSAGE_ERROR, summary_message=msg, detailed_message="The synchronization was performed partially." ) result = { "error_message": msg } except CiscoApiCallFailed as ex: msg = "Server unreachable (%s)" % str(ex) logger.error(msg, exc_info=True) NotificationMessage.objects.create( title="Synchronization with Cisco EoX API", type=NotificationMessage.MESSAGE_ERROR, summary_message=msg, detailed_message="The synchronization was performed partially." ) result = { "error_message": msg } else: result = { "status_message": "task not enabled" } # remove in progress flag with the cache cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS") return result
def execute_task_to_synchronize_cisco_eox_states( self, ignore_periodic_sync_flag=False): """ This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the results in the local database. There are two types of operation: * cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the blacklist and not in the database * cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already included in the database :return: """ app_config = AppSettings() run_task = app_config.is_periodic_sync_enabled() if run_task or ignore_periodic_sync_flag: logger.info("start sync with Cisco EoX API...") self.update_state( state=TaskState.PROCESSING, meta={"status_message": "sync with Cisco EoX API..."}) # read queries from configuration queries = app_config.get_cisco_eox_api_queries_as_list() if len(queries) == 0: result = {"status_message": "No Cisco EoX API queries configured."} NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_WARNING, summary_message= "There are no Cisco EoX API queries configured. Nothing to do.", detailed_message= "There are no Cisco EoX API queries configured. Please configure at least on EoX API " "query in the settings or disable the periodic synchronization." ) # update the local database with the Cisco EoX API else: try: # test Cisco EoX API access success = utils.check_cisco_eox_api_access( app_config.get_cisco_api_client_id(), app_config.get_cisco_api_client_secret(), False) if not success: msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \ "and that the authentication settings are valid." logger.error(msg, exc_info=True) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR, summary_message=msg, detailed_message= "The synchronization with the Cisco EoX API was not started." ) result = {"error_message": msg} else: # execute all queries from the configuration and collect the results notify_metrics = {"queries": {}} counter = 0 for query in queries: self.update_state( state=TaskState.PROCESSING, meta={ "status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of " "%d</strong>)..." % (query, counter + 1, len(queries)) }) # wait a specific amount of seconds between each update call time.sleep( int(app_config.get_cisco_eox_api_sync_wait_time())) query_results = cisco_eox_api_crawler.update_cisco_eox_database( query) blist_counter = 0 update_counter = 0 create_counter = 0 for qres in query_results: if qres["created"]: create_counter += 1 elif qres["updated"]: update_counter += 1 elif qres["blacklist"]: blist_counter += 1 notify_metrics["queries"][query] = { "amount": len(query_results), "updated_entries": update_counter, "created_entries": create_counter, "blacklisted_entries": blist_counter, "result_details": query_results } counter += 1 # create NotificationMessage based on the results detailed_html = "" blist_counter = 0 update_counter = 0 create_counter = 0 for query_key in notify_metrics["queries"].keys(): update_counter += notify_metrics["queries"][query_key][ "updated_entries"] create_counter += notify_metrics["queries"][query_key][ "created_entries"] blist_counter += notify_metrics["queries"][query_key][ "blacklisted_entries"] # build detailed string detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key cond_1 = notify_metrics["queries"][query_key][ "updated_entries"] == 0 cond_1 = cond_1 and (notify_metrics["queries"] [query_key]["created_entries"] == 0) cond_1 = cond_1 and (notify_metrics["queries"] [query_key]["blacklisted_entries"] == 0) if cond_1: detailed_html += "No changes required." else: detailed_html += "The following products are affected by this update:</p>" detailed_html += "<ul>" for qres in notify_metrics["queries"][query_key][ "result_details"]: msg = "" if "message" in qres.keys(): if qres["message"]: msg = qres["message"] if qres["created"]: detailed_html += "<li>create the Product <code>%s</code> in the database" % ( qres["PID"]) detailed_html += " (%s)</li>" % msg if msg != "" else "</li>" elif qres["updated"]: detailed_html += "<li>update the Product data for <code>%s</code>" % ( qres["PID"]) detailed_html += " (%s)</li>" % msg if msg != "" else "</li>" elif qres["blacklist"]: detailed_html += "<li>Product data for <code>%s</code> ignored" % ( qres["PID"]) detailed_html += " (%s)</li>" % msg if msg != "" else "</li>" detailed_html += "</ul>" detailed_html += "</div>" summary_html = "The synchronization was performed successfully. " if update_counter == 1: summary_html += "<strong>%d</strong> product was updated, " % update_counter else: summary_html += "<strong>%d</strong> products are updated, " % update_counter if create_counter == 1: summary_html += "<strong>%s</strong> product was added to the database and " % create_counter else: summary_html += "<strong>%s</strong> products are added to the database and " % create_counter if blist_counter == 1: summary_html += "<strong>%d</strong> product was ignored." % blist_counter else: summary_html += "<strong>%d</strong> products are ignored." % blist_counter # show the executed queries in the summary message summary_html += " The following queries were executed: %s" % ", ".join( ["<code>%s</code>" % query for query in queries]) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_SUCCESS, summary_message=summary_html, detailed_message=detailed_html) result = {"status_message": detailed_html} # if the task was executed eager, set state to SUCCESS (required for testing) if self.request.is_eager: self.update_state( state=TaskState.SUCCESS, meta={"status_message": detailed_html}) except CredentialsNotFoundException as ex: msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str( ex) logger.error(msg, exc_info=True) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR, summary_message=msg, detailed_message= "The synchronization was performed partially.") result = {"error_message": msg} except CiscoApiCallFailed as ex: msg = "Cisco EoX API call failed (%s)" % str(ex) logger.error(msg, exc_info=True) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR, summary_message=msg, detailed_message= "The synchronization was performed partially.") result = {"error_message": msg} except Exception as ex: msg = "Cannot access the Cisco API. Please ensure that the server is " \ "connected to the internet and that the authentication settings are " \ "valid." logger.error(msg, exc_info=True) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR, summary_message=msg, detailed_message="%s" % str(ex)) result = {"error_message": msg} else: result = {"status_message": "task not enabled"} # remove in progress flag with the cache cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS") return result
def test_cisco_eox_database_query_with_disabled_cisco_eox_api(self): with pytest.raises(CiscoApiCallFailed) as exinfo: api_crawler.update_cisco_eox_database("MyQuery") assert exinfo.match("Cisco API access not enabled")
def test_invalid_cisco_eox_database_query(self): with pytest.raises(ValueError) as exinfo: api_crawler.update_cisco_eox_database(None) assert exinfo.match("api_query must be a string value")