def test_cisco_eox_database_query_with_server_error(self, monkeypatch):
def raise_error():
raise Exception("Server is down")
monkeypatch.setattr(requests, "get", raise_error)
with pytest.raises(ConnectionFailedException) as exinfo:
api_crawler.update_cisco_eox_database("MyQuery")
assert exinfo.match("cannot contact API endpoint at")
def test_offline_invalid_update_cisco_eox_database_with_default_settings(self, monkeypatch):
# mock the underlying GET request
def mock_response():
r = Response()
r.status_code = 200
with open("app/ciscoeox/tests/data/cisco_eox_error_response.json") as f:
r._content = f.read().encode("utf-8")
return r
monkeypatch.setattr(requests, "get", lambda x, headers: mock_response())
with pytest.raises(CiscoApiCallFailed) as exinfo:
api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS")
assert exinfo.match("Cisco EoX API error: Some unknown error occurred during the API access \(YXCABC\)")
def test_offline_valid_update_cisco_eox_database_with_multiple_urls_in_api_reponse(self, monkeypatch):
# mock the underlying GET request
def mock_response():
r = Response()
r.status_code = 200
with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f:
content = f.read()
# modify the URL parameter in the migration data (every import should fail)
json_content = json.loads(content)
values = [
"http://localhost;",
"http://localhost;",
"http://localhost and http://another-localhost",
]
for c in range(0, len(json_content["EOXRecord"])):
json_content["EOXRecord"][c]["EOXMigrationDetails"]["MigrationProductInfoURL"] = values[c % 3]
r._content = json.dumps(json_content).encode("utf-8")
return r
monkeypatch.setattr(requests, "get", lambda x, headers: mock_response())
result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS")
assert len(result) == 3, "Three products should be detected"
# every body should contain a specific message
for e in result:
assert "message" in e
assert e["message"] == "multiple URL values received, only the first one is saved"
assert Product.objects.count() == 3, "Products are created"
assert ProductMigrationOption.objects.count() == 3, "Migration data are created"
def test_offline_no_result_update_cisco_eox_database_with_create_flag(self, monkeypatch):
# mock the underlying GET request
def mock_response():
r = Response()
r.status_code = 200
with open("app/ciscoeox/tests/data/cisco_eox_no_result_response.json") as f:
r._content = f.read().encode("utf-8")
return r
monkeypatch.setattr(requests, "get", lambda x, headers: mock_response())
result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS")
assert len(result) == 1, "Three products should be imported"
assert "blacklist" in result[0]
assert "updated" in result[0]
assert "created" in result[0]
assert "PID" in result[0]
assert "message" in result[0]
assert result[0]["blacklist"] is False
assert result[0]["updated"] is False
assert result[0]["created"] is False
assert result[0]["PID"] is None
assert result[0]["message"] == "No product update required"
assert Product.objects.count() == 0, "No products are created, because the creation mode is disabled by default"
def test_offline_valid_update_cisco_eox_database_with_multiple_urls_in_result(self, monkeypatch):
# mock the underlying GET request
def mock_response():
r = Response()
r.status_code = 200
with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f:
raw_data = f.read()
jdata = json.loads(raw_data)
jdata["EOXRecord"][0]["EOLProductID"] = "TEST_PID"
jdata["EOXRecord"][0]["LinkToProductBulletinURL"] = "http://somewhere.com/index.html ," \
"https://other.com/index.html"
r._content = json.dumps(jdata).encode("utf-8")
return r
monkeypatch.setattr(requests, "get", lambda x, headers: mock_response())
result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS")
assert len(result) == 3, "Three products should be seen in the API response"
assert Product.objects.count() == 3, "Three products should be imported to the database"
p = Product.objects.get(product_id="TEST_PID")
assert p.eol_reference_url == "http://somewhere.com/index.html", "only the first entry is stored in the " \
"database"
def test_offline_valid_update_cisco_eox_database_with_default_settings(self, monkeypatch):
# mock the underlying GET request
def mock_response():
r = Response()
r.status_code = 200
with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f:
r._content = f.read().encode("utf-8")
return r
monkeypatch.setattr(requests, "get", lambda x, headers: mock_response())
result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS")
assert len(result) == 3, "Three products should be imported"
# every product should contain the following values
for e in result:
assert "blacklist" in e
assert "updated" in e
assert "created" in e
assert "PID" in e
assert "message" in e
assert e["blacklist"] is False
assert e["updated"] is False
assert e["created"] is False
assert type(e["PID"]) == str
assert e["message"] is None
assert Product.objects.count() == 0, "No products are created, because the creation mode is disabled by default"
def test_offline_valid_update_cisco_eox_database_with_create_flag(self, monkeypatch):
# mock the underlying GET request
def mock_response():
r = Response()
r.status_code = 200
with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f:
r._content = f.read().encode("utf-8")
return r
monkeypatch.setattr(requests, "get", lambda x, headers: mock_response())
result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS")
assert len(result) == 3, "Three products should be imported"
# every product should contain the following values
for e in result:
assert "blacklist" in e
assert "updated" in e
assert "created" in e
assert "PID" in e
assert "message" in e
assert e["blacklist"] is False
assert e["updated"] is True
assert e["created"] is True
assert type(e["PID"]) == str
assert e["message"] is None
assert Product.objects.count() == 3, "Products from the test API results are created"
# call the same query again
result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS")
for e in result:
assert "blacklist" in e
assert "updated" in e
assert "created" in e
assert "PID" in e
assert "message" in e
assert e["blacklist"] is False
assert e["updated"] is False
assert e["created"] is False
assert type(e["PID"]) == str
assert e["message"] == "update suppressed (data not modified)"
assert len(result) == 3, "Nothing should change"
def cisco_eox_query(request):
"""Manual query page against the Cisco EoX Version 5 API (if enabled)
:param request:
:return:
"""
app_config = AppSettings()
app_config.read_file()
cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {"is_cisco_api_enabled": cisco_api_enabled}
if request.method == "POST":
# create a form instance and populate it with data from the request:
if "sync_cisco_eox_states_now" in request.POST.keys():
if "sync_cisco_eox_states_query" in request.POST.keys():
query = request.POST['sync_cisco_eox_states_query']
if query != "":
if len(query.split(" ")) == 1:
context['query_executed'] = query
try:
eox_api_update_records = update_cisco_eox_database(
api_query=query)
except ConnectionFailedException as ex:
eox_api_update_records = [
"Cannot contact Cisco API, error message:\n%s"
% ex
]
except CiscoApiCallFailed as ex:
eox_api_update_records = [ex]
except Exception as ex:
logger.debug(
"execution failed due to unexpected exception",
exc_info=True)
eox_api_update_records = [
"execution failed: %s" % ex
]
context['eox_api_update_records'] = json.dumps(
eox_api_update_records, indent=4, sort_keys=True)
else:
context['eox_api_update_records'] = "Invalid query '%s': not executed" % \
request.POST['sync_cisco_eox_states_query']
else:
context['eox_api_update_records'] = [
"Please specify a valid query"
]
else:
context[
'eox_api_update_records'] = "Query not executed, please select the \"execute it now\" checkbox."
return render(request, "ciscoeox/cisco_eox_query.html", context=context)
def cisco_eox_query(request):
"""Manual query page against the Cisco EoX Version 5 API (if enabled)
:param request:
:return:
"""
app_config = AppSettings()
cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {
"is_cisco_api_enabled": cisco_api_enabled
}
if request.method == "POST":
# create a form instance and populate it with data from the request:
if "sync_cisco_eox_states_now" in request.POST.keys():
if "sync_cisco_eox_states_query" in request.POST.keys():
query = request.POST['sync_cisco_eox_states_query']
if query != "":
if len(query.split(" ")) == 1:
context['query_executed'] = query
try:
eox_api_update_records = api_crawler.update_cisco_eox_database(api_query=query)
except ConnectionFailedException as ex:
eox_api_update_records = {"error": "Cannot contact Cisco API, error message:\n%s" % ex}
except CiscoApiCallFailed as ex:
eox_api_update_records = {"error": "Cisco API call failed: %s" % ex}
except Exception as ex: # catch any exception
logger.debug("execution failed due to unexpected exception", exc_info=True)
eox_api_update_records = ["execution failed: %s" % ex]
context['eox_api_update_records'] = json.dumps(eox_api_update_records, indent=4, sort_keys=True)
else:
context['eox_api_update_records'] = "Invalid query '%s': not executed" % \
request.POST['sync_cisco_eox_states_query']
else:
context['eox_api_update_records'] = ["Please specify a valid query"]
else:
context['eox_api_update_records'] = "Query not specified."
else:
context['eox_api_update_records'] = "Query not executed, please select the \"execute it now\" checkbox."
return render(request, "ciscoeox/cisco_eox_query.html", context=context)
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False):
"""
This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
results in the local database. There are two types of operation:
* cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
blacklist and not in the database
* cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
included in the database
:return:
"""
app_config = AppSettings()
app_config.read_file()
run_task = app_config.is_cisco_eox_api_auto_sync_enabled()
if ignore_periodic_sync_flag:
run_task = True
if run_task:
logger.info("start sync with Cisco EoX API...")
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "sync with Cisco EoX API..."
})
# read queries from configuration
queries = app_config.get_cisco_eox_api_queries().splitlines()
if len(queries) == 0:
result = {
"status_message": "No Cisco EoX API queries configured."
}
# update the local database with the Cisco EoX API
else:
# test Cisco EoX API access
success, _ = test_cisco_eox_api_access(
app_config.get_cisco_api_client_id(),
app_config.get_cisco_api_client_secret(),
False
)
if not success:
msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \
"and that the authentication settings are valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization with the Cisco EoX API was not started."
)
result = {
"error_message": msg
}
else:
try:
# execute query by query
notify_metrics = {
"queries": {}
}
counter = 0
for query in queries:
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
"%d</strong>)..." % (query, counter + 1, len(queries))
})
query_results = cisco_eox_api_crawler.update_cisco_eox_database(query)
blist_counter = 0
update_counter = 0
create_counter = 0
for qres in query_results:
if qres["created"]:
create_counter += 1
elif qres["updated"]:
update_counter += 1
elif qres["blacklist"]:
blist_counter += 1
notify_metrics["queries"][query] = {
"amount": len(query_results),
"updated_entries": update_counter,
"created_entries": create_counter,
"blacklisted_entries": blist_counter,
"result_details": query_results
}
counter += 1
# create NotificationMessage
detailed_html = ""
blist_counter = 0
update_counter = 0
create_counter = 0
for query_key in notify_metrics["queries"].keys():
update_counter += notify_metrics["queries"][query_key]["updated_entries"]
create_counter += notify_metrics["queries"][query_key]["created_entries"]
blist_counter += notify_metrics["queries"][query_key]["blacklisted_entries"]
# build detailed string
detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key
cond_1 = notify_metrics["queries"][query_key]["updated_entries"] == 0
cond_1 = cond_1 and (notify_metrics["queries"][query_key]["created_entries"] == 0)
cond_1 = cond_1 and (notify_metrics["queries"][query_key]["blacklisted_entries"] == 0)
if cond_1:
detailed_html += "No changes required."
else:
detailed_html += "The following products are affected by this update:</p>"
detailed_html += "<ul>"
for qres in notify_metrics["queries"][query_key]["result_details"]:
msg = ""
if "message" in qres.keys():
if qres["message"]:
msg = qres["message"]
if qres["created"]:
detailed_html += "<li>create the Product <code>%s</code> in the database" % (
qres["PID"]
)
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
elif qres["updated"]:
detailed_html += "<li>update the Product data for <code>%s</code></li>" % (
qres["PID"]
)
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
elif qres["blacklist"]:
detailed_html += "<li>Product data for <code>%s</code> ignored</li>" % (
qres["PID"]
)
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
detailed_html += "</ul>"
detailed_html += "</div>"
summary_html = "The synchronization was performed successfully. "
if update_counter == 1:
summary_html += "<strong>%d</strong> product was updated, " % update_counter
else:
summary_html += "<strong>%d</strong> products are updated, " % update_counter
if create_counter == 1:
summary_html += "<strong>%s</strong> product was added to the database and " % create_counter
else:
summary_html += "<strong>%s</strong> products are added to the database and " % create_counter
if blist_counter == 1:
summary_html += "<strong>%d</strong> product was ignored." % blist_counter
else:
summary_html += "<strong>%d</strong> products are ignored." % blist_counter
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_SUCCESS,
summary_message=summary_html,
detailed_message=detailed_html
)
result = {
"status_message": detailed_html
}
# if the task was executed eager, set state to SUCCESS (required for testing)
if self.request.is_eager:
self.update_state(state=TaskState.SUCCESS, meta={
"status_message": detailed_html
})
except CredentialsNotFoundException as ex:
msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization was performed partially."
)
result = {
"error_message": msg
}
except CiscoApiCallFailed as ex:
msg = "Server unreachable (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization was performed partially."
)
result = {
"error_message": msg
}
else:
result = {
"status_message": "task not enabled"
}
# remove in progress flag with the cache
cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")
return result
def execute_task_to_synchronize_cisco_eox_states(
self, ignore_periodic_sync_flag=False):
"""
This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
results in the local database. There are two types of operation:
* cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
blacklist and not in the database
* cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
included in the database
:return:
"""
app_config = AppSettings()
run_task = app_config.is_periodic_sync_enabled()
if run_task or ignore_periodic_sync_flag:
logger.info("start sync with Cisco EoX API...")
self.update_state(
state=TaskState.PROCESSING,
meta={"status_message": "sync with Cisco EoX API..."})
# read queries from configuration
queries = app_config.get_cisco_eox_api_queries_as_list()
if len(queries) == 0:
result = {"status_message": "No Cisco EoX API queries configured."}
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_WARNING,
summary_message=
"There are no Cisco EoX API queries configured. Nothing to do.",
detailed_message=
"There are no Cisco EoX API queries configured. Please configure at least on EoX API "
"query in the settings or disable the periodic synchronization."
)
# update the local database with the Cisco EoX API
else:
try:
# test Cisco EoX API access
success = utils.check_cisco_eox_api_access(
app_config.get_cisco_api_client_id(),
app_config.get_cisco_api_client_secret(), False)
if not success:
msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \
"and that the authentication settings are valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization with the Cisco EoX API was not started."
)
result = {"error_message": msg}
else:
# execute all queries from the configuration and collect the results
notify_metrics = {"queries": {}}
counter = 0
for query in queries:
self.update_state(
state=TaskState.PROCESSING,
meta={
"status_message":
"send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
"%d</strong>)..." %
(query, counter + 1, len(queries))
})
# wait a specific amount of seconds between each update call
time.sleep(
int(app_config.get_cisco_eox_api_sync_wait_time()))
query_results = cisco_eox_api_crawler.update_cisco_eox_database(
query)
blist_counter = 0
update_counter = 0
create_counter = 0
for qres in query_results:
if qres["created"]:
create_counter += 1
elif qres["updated"]:
update_counter += 1
elif qres["blacklist"]:
blist_counter += 1
notify_metrics["queries"][query] = {
"amount": len(query_results),
"updated_entries": update_counter,
"created_entries": create_counter,
"blacklisted_entries": blist_counter,
"result_details": query_results
}
counter += 1
# create NotificationMessage based on the results
detailed_html = ""
blist_counter = 0
update_counter = 0
create_counter = 0
for query_key in notify_metrics["queries"].keys():
update_counter += notify_metrics["queries"][query_key][
"updated_entries"]
create_counter += notify_metrics["queries"][query_key][
"created_entries"]
blist_counter += notify_metrics["queries"][query_key][
"blacklisted_entries"]
# build detailed string
detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key
cond_1 = notify_metrics["queries"][query_key][
"updated_entries"] == 0
cond_1 = cond_1 and (notify_metrics["queries"]
[query_key]["created_entries"]
== 0)
cond_1 = cond_1 and (notify_metrics["queries"]
[query_key]["blacklisted_entries"]
== 0)
if cond_1:
detailed_html += "No changes required."
else:
detailed_html += "The following products are affected by this update:</p>"
detailed_html += "<ul>"
for qres in notify_metrics["queries"][query_key][
"result_details"]:
msg = ""
if "message" in qres.keys():
if qres["message"]:
msg = qres["message"]
if qres["created"]:
detailed_html += "<li>create the Product <code>%s</code> in the database" % (
qres["PID"])
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
elif qres["updated"]:
detailed_html += "<li>update the Product data for <code>%s</code>" % (
qres["PID"])
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
elif qres["blacklist"]:
detailed_html += "<li>Product data for <code>%s</code> ignored" % (
qres["PID"])
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
detailed_html += "</ul>"
detailed_html += "</div>"
summary_html = "The synchronization was performed successfully. "
if update_counter == 1:
summary_html += "<strong>%d</strong> product was updated, " % update_counter
else:
summary_html += "<strong>%d</strong> products are updated, " % update_counter
if create_counter == 1:
summary_html += "<strong>%s</strong> product was added to the database and " % create_counter
else:
summary_html += "<strong>%s</strong> products are added to the database and " % create_counter
if blist_counter == 1:
summary_html += "<strong>%d</strong> product was ignored." % blist_counter
else:
summary_html += "<strong>%d</strong> products are ignored." % blist_counter
# show the executed queries in the summary message
summary_html += " The following queries were executed: %s" % ", ".join(
["<code>%s</code>" % query for query in queries])
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_SUCCESS,
summary_message=summary_html,
detailed_message=detailed_html)
result = {"status_message": detailed_html}
# if the task was executed eager, set state to SUCCESS (required for testing)
if self.request.is_eager:
self.update_state(
state=TaskState.SUCCESS,
meta={"status_message": detailed_html})
except CredentialsNotFoundException as ex:
msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(
ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization was performed partially.")
result = {"error_message": msg}
except CiscoApiCallFailed as ex:
msg = "Cisco EoX API call failed (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization was performed partially.")
result = {"error_message": msg}
except Exception as ex:
msg = "Cannot access the Cisco API. Please ensure that the server is " \
"connected to the internet and that the authentication settings are " \
"valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="%s" % str(ex))
result = {"error_message": msg}
else:
result = {"status_message": "task not enabled"}
# remove in progress flag with the cache
cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")
return result
def test_cisco_eox_database_query_with_disabled_cisco_eox_api(self):
with pytest.raises(CiscoApiCallFailed) as exinfo:
api_crawler.update_cisco_eox_database("MyQuery")
assert exinfo.match("Cisco API access not enabled")
def test_invalid_cisco_eox_database_query(self):
with pytest.raises(ValueError) as exinfo:
api_crawler.update_cisco_eox_database(None)
assert exinfo.match("api_query must be a string value")
