def cisco_eox_query(request):
"""Manual query page against the Cisco EoX Version 5 API (if enabled)
:param request:
:return:
"""
app_config = AppSettings()
app_config.read_file()
cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {"is_cisco_api_enabled": cisco_api_enabled}
if request.method == "POST":
# create a form instance and populate it with data from the request:
if "sync_cisco_eox_states_now" in request.POST.keys():
if "sync_cisco_eox_states_query" in request.POST.keys():
query = request.POST['sync_cisco_eox_states_query']
if query != "":
if len(query.split(" ")) == 1:
context['query_executed'] = query
try:
eox_api_update_records = update_cisco_eox_database(
api_query=query)
except ConnectionFailedException as ex:
eox_api_update_records = [
"Cannot contact Cisco API, error message:\n%s"
% ex
]
except CiscoApiCallFailed as ex:
eox_api_update_records = [ex]
except Exception as ex:
logger.debug(
"execution failed due to unexpected exception",
exc_info=True)
eox_api_update_records = [
"execution failed: %s" % ex
]
context['eox_api_update_records'] = json.dumps(
eox_api_update_records, indent=4, sort_keys=True)
else:
context['eox_api_update_records'] = "Invalid query '%s': not executed" % \
request.POST['sync_cisco_eox_states_query']
else:
context['eox_api_update_records'] = [
"Please specify a valid query"
]
else:
context[
'eox_api_update_records'] = "Query not executed, please select the \"execute it now\" checkbox."
return render(request, "ciscoeox/cisco_eox_query.html", context=context)
def load_client_credentials(self):
logger.debug("load client credentials from configuration")
app_settings = AppSettings()
app_settings.read_file()
# load client credentials
self.client_id = app_settings.get_cisco_api_client_id()
self.client_secret = app_settings.get_cisco_api_client_secret()
def test_write_datetime_to_settings_file(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
settings.create_defaults()
# get a configuration value (default in the global section)
now = datetime.now()
value = now.isoformat()
settings.set(
key="cisco_eox_api_auto_sync_last_execution_time",
section=AppSettings.CISCO_EOX_CRAWLER_SECTION,
value=value
)
settings.write_file()
read_value = settings.get_string(
key="cisco_eox_api_auto_sync_last_execution_time",
section=AppSettings.CISCO_EOX_CRAWLER_SECTION
)
self.assertEqual(value, read_value)
self.assertEqual(now, parse_datetime(read_value))
# cleanup
self.clean_config_file()
def cisco_eox_query(request):
"""Manual query page against the Cisco EoX Version 5 API (if enabled)
:param request:
:return:
"""
app_config = AppSettings()
app_config.read_file()
cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {
"is_cisco_api_enabled": cisco_api_enabled
}
if request.method == "POST":
# create a form instance and populate it with data from the request:
if "sync_cisco_eox_states_now" in request.POST.keys():
if "sync_cisco_eox_states_query" in request.POST.keys():
query = request.POST['sync_cisco_eox_states_query']
if query != "":
if len(query.split(" ")) == 1:
context['query_executed'] = query
try:
eox_api_update_records = update_cisco_eox_database(api_query=query)
except ConnectionFailedException as ex:
eox_api_update_records = ["Cannot contact Cisco API, error message:\n%s" % ex]
except CiscoApiCallFailed as ex:
eox_api_update_records = [ex]
except Exception as ex:
logger.debug("execution failed due to unexpected exception", exc_info=True)
eox_api_update_records = ["execution failed: %s" % ex]
context['eox_api_update_records'] = json.dumps(eox_api_update_records, indent=4, sort_keys=True)
else:
context['eox_api_update_records'] = "Invalid query '%s': not executed" % \
request.POST['sync_cisco_eox_states_query']
else:
context['eox_api_update_records'] = ["Please specify a valid query"]
else:
context['eox_api_update_records'] = "Query not executed, please select the \"execute it now\" checkbox."
return render(request, "ciscoeox/cisco_eox_query.html", context=context)
def test_read_with_missing_file(self):
"""
test the read function if no file exists
"""
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
# read file specified in the settings
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# cleanup
self.clean_config_file()
def test_read_with_missing_file(self):
"""
test the read function if no file exists
"""
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
# read file specified in the settings
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# cleanup
self.clean_config_file()
def test_eox_update_call_with_sample_data(self):
app_config = AppSettings()
app_config.read_file()
app_config.set_cisco_api_enabled(True)
app_config.set_periodic_sync_enabled(True)
app_config.write_file()
eox_sample_response = os.path.join("app",
"ciscoeox",
"tests",
"cisco_eox_sample_response.json")
eox_db_json = json.loads(open(eox_sample_response).read())
for record in eox_db_json['EOXRecord']:
api_crawler.update_local_db_based_on_record(record, True)
def update_cisco_eox_database(api_query):
"""
Synchronizes the local EoX data from the Cisco EoX API using the specified queries or the queries specified in the
configuration when api_query is set to None
:param api_query: single query that is send to the Cisco EoX API
:return:
"""
# load application settings and check, that the API is enabled
app_settings = AppSettings()
app_settings.read_file()
if not app_settings.is_cisco_api_enabled():
msg = "Cisco API access not enabled"
logger.warn(msg)
raise CiscoApiCallFailed(msg)
blacklist = app_settings.get_product_blacklist_regex().split(";")
create_missing = app_settings.is_auto_create_new_products()
# start with Cisco EoX API queries
logger.info("Query EoX database: %s" % api_query)
query_results = query_cisco_eox_api(api_query, blacklist, create_missing)
# filter empty result strings
if len(query_results) == 0:
query_results = [{
"PID": None,
"blacklist": False,
"updated": False,
"created": False,
"message": "No product update required"
}]
return query_results
def test_create_default_config(self):
"""
test the create of the default configuration
"""
self.clean_config_file()
# create new AppSettings object and create defaults
settings = AppSettings()
settings.create_defaults()
# verify result
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# test dictionary (booleans are converted as string)
self.assertEqual(json.dumps(settings.CONFIG_DEFAULTS, sort_keys=True),
json.dumps(settings.to_dictionary(), sort_keys=True))
# cleanup
self.clean_config_file()
def test_trigger_manual_cisco_eox_synchronization(self):
"""
Test if a Cisco EoX synchronization task is scheduled after
:return:
"""
app_config = AppSettings()
app_config.read_file()
app_config.set_cisco_api_enabled(True)
app_config.set_periodic_sync_enabled(True)
app_config.write_file()
# schedule Cisco EoX API update
url = reverse('cisco_api:start_cisco_eox_api_sync_now')
self.client.login(username="******", password="******")
resp = self.client.get(url)
self.assertEqual(resp.status_code, 302)
# verify that task ID is saved in the cache (set by the schedule call)
task_id = cache.get("CISCO_EOX_API_SYN_IN_PROGRESS", "")
self.assertNotEqual(task_id, "")
def login_required_if_login_only_mode(request):
"""
Test if the login only mode is enabled. If this is the case, test if a user is authentication. If this is not the
case, redirect to the login form.
"""
login_only_mode = cache.get("LOGIN_ONLY_MODE_SETTING", None)
if not login_only_mode:
# key not in cache
app_settings = AppSettings()
app_settings.read_file()
login_only_mode = app_settings.is_login_only_mode()
# add setting to cache
cache.set("LOGIN_ONLY_MODE_SETTING", login_only_mode, 60 * 60)
if login_only_mode:
if not request.user.is_authenticated():
return True
return False
def test_create_default_config(self):
"""
test the create of the default configuration
"""
self.clean_config_file()
# create new AppSettings object and create defaults
settings = AppSettings()
settings.create_defaults()
# verify result
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# test dictionary (booleans are converted as string)
self.assertEqual(
json.dumps(settings.CONFIG_DEFAULTS, sort_keys=True),
json.dumps(settings.to_dictionary(), sort_keys=True)
)
# cleanup
self.clean_config_file()
def test_auto_create_new_products(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# get values
value = settings.is_auto_create_new_products()
self.assertEqual(value, False)
# set values
settings.set_auto_create_new_products(True)
settings.write_file()
value = settings.is_auto_create_new_products()
self.assertEqual(value, True)
# cleanup
self.clean_config_file()
def load_client_credentials(self):
logger.debug("load client credentials from configuration")
app_settings = AppSettings()
app_settings.read_file()
# load client credentials
self.client_id = app_settings.get_cisco_api_client_id()
self.client_secret = app_settings.get_cisco_api_client_secret()
def test_get_cisco_api_client_secret(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# get values
value = settings.get_cisco_api_client_secret()
self.assertEqual(value, "PlsChgMe")
# set values
value = "test_secret"
settings.set_cisco_api_client_secret(value)
settings.write_file()
value = settings.get_cisco_api_client_secret()
self.assertEqual(value, value)
# cleanup
self.clean_config_file()
def test_get_cisco_eox_queries_from_config(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# get values
value = settings.get_cisco_eox_api_queries()
self.assertEqual(value, "")
# set values
queries = "test"
settings.set_cisco_eox_api_queries(queries)
settings.write_file()
value = settings.get_cisco_eox_api_queries()
self.assertEqual(value, queries)
# cleanup
self.clean_config_file()
def test_get_product_blacklist_regex_from_config(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# get values
value = settings.get_product_blacklist_regex()
self.assertEqual(value, "")
# set values
blacklist_entries = "test"
settings.set_product_blacklist_regex(blacklist_entries)
settings.write_file()
value = settings.get_product_blacklist_regex()
self.assertEqual(value, blacklist_entries)
# cleanup
self.clean_config_file()
def test_auto_create_new_products(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# get values
value = settings.is_auto_create_new_products()
self.assertEqual(value, False)
# set values
settings.set_auto_create_new_products(True)
settings.write_file()
value = settings.is_auto_create_new_products()
self.assertEqual(value, True)
# cleanup
self.clean_config_file()
def test_get_cisco_api_client_secret(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# get values
value = settings.get_cisco_api_client_secret()
self.assertEqual(value, "PlsChgMe")
# set values
value = "test_secret"
settings.set_cisco_api_client_secret(value)
settings.write_file()
value = settings.get_cisco_api_client_secret()
self.assertEqual(value, value)
# cleanup
self.clean_config_file()
def test_get_cisco_eox_queries_from_config(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# get values
value = settings.get_cisco_eox_api_queries()
self.assertEqual(value, "")
# set values
queries = "test"
settings.set_cisco_eox_api_queries(queries)
settings.write_file()
value = settings.get_cisco_eox_api_queries()
self.assertEqual(value, queries)
# cleanup
self.clean_config_file()
def test_get_product_blacklist_regex_from_config(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
self.assertFalse(os.path.exists(self.TEST_CONFIG_FILE))
settings.read_file()
self.assertTrue(os.path.exists(self.TEST_CONFIG_FILE))
# get values
value = settings.get_product_blacklist_regex()
self.assertEqual(value, "")
# set values
blacklist_entries = "test"
settings.set_product_blacklist_regex(blacklist_entries)
settings.write_file()
value = settings.get_product_blacklist_regex()
self.assertEqual(value, blacklist_entries)
# cleanup
self.clean_config_file()
def test_write_datetime_to_settings_file(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
settings.create_defaults()
# get a configuration value (default in the global section)
now = datetime.now()
value = now.isoformat()
settings.set(key="cisco_eox_api_auto_sync_last_execution_time",
section=AppSettings.CISCO_EOX_CRAWLER_SECTION,
value=value)
settings.write_file()
read_value = settings.get_string(
key="cisco_eox_api_auto_sync_last_execution_time",
section=AppSettings.CISCO_EOX_CRAWLER_SECTION)
self.assertEqual(value, read_value)
self.assertEqual(now, parse_datetime(read_value))
# cleanup
self.clean_config_file()
def change_configuration(request):
"""
change configuration of the Product Database
"""
app_config = AppSettings()
app_config.read_file()
if request.method == "POST":
# create a form instance and populate it with data from the request:
form = SettingsForm(request.POST)
if form.is_valid():
# set common settings
app_config.set_login_only_mode(form.cleaned_data["login_only_mode"])
# set the Cisco API configuration options
api_enabled = form.cleaned_data["cisco_api_enabled"]
if not api_enabled:
# api is disabled, reset values to default
app_config.set_cisco_api_enabled(api_enabled)
app_config.set_cisco_api_client_id("PlsChgMe")
app_config.set_cisco_api_client_secret("PlsChgMe")
app_config.set_cisco_eox_api_auto_sync_enabled(False)
app_config.set_auto_create_new_products(False)
app_config.set_cisco_eox_api_queries("")
app_config.set_product_blacklist_regex("")
else:
app_config.set_cisco_api_enabled(api_enabled)
client_id = form.cleaned_data["cisco_api_client_id"] \
if form.cleaned_data["cisco_api_client_id"] != "" else "PlsChgMe"
app_config.set_cisco_api_client_id(client_id)
client_secret = form.cleaned_data["cisco_api_client_secret"] \
if form.cleaned_data["cisco_api_client_secret"] != "" else "PlsChgMe"
app_config.set_cisco_api_client_secret(client_secret)
if client_id != "PlsChgMe":
if settings.DEMO_MODE:
messages.success(request, "Successfully connected to the Cisco EoX API (Demo Mode)")
else:
result, message = test_cisco_eox_api_access(
form.cleaned_data["cisco_api_client_id"],
form.cleaned_data["cisco_api_client_secret"]
)
if result:
messages.success(request, "Successfully connected to the Cisco EoX API")
else:
messages.error(request, "Cannot contact the Cisco EoX API: %s" % message)
else:
messages.info(
request,
"Please configure your Cisco API credentials within the Cisco API settings tab."
)
app_config.set_cisco_eox_api_auto_sync_enabled(form.cleaned_data["eox_api_auto_sync_enabled"])
app_config.set_auto_create_new_products(form.cleaned_data["eox_auto_sync_auto_create_elements"])
app_config.set_cisco_eox_api_queries(form.cleaned_data["eox_api_queries"])
app_config.set_product_blacklist_regex(form.cleaned_data["eox_api_blacklist"])
app_config.write_file()
# expire cached settings
cache.delete("LOGIN_ONLY_MODE_SETTING")
messages.success(request, "Settings saved successfully")
return redirect(resolve_url("productdb_config:change_settings"))
else:
form = SettingsForm()
form.fields['cisco_api_enabled'].initial = app_config.is_cisco_api_enabled()
form.fields['login_only_mode'].initial = app_config.is_login_only_mode()
form.fields['cisco_api_client_id'].initial = app_config.get_cisco_api_client_id()
form.fields['cisco_api_client_secret'].initial = app_config.get_cisco_api_client_secret()
form.fields['eox_api_auto_sync_enabled'].initial = app_config.is_cisco_eox_api_auto_sync_enabled()
form.fields['eox_auto_sync_auto_create_elements'].initial = app_config.is_auto_create_new_products()
form.fields['eox_api_queries'].initial = app_config.get_cisco_eox_api_queries()
form.fields['eox_api_blacklist'].initial = app_config.get_product_blacklist_regex()
context = {
"form": form,
"is_cisco_api_enabled": app_config.is_cisco_api_enabled(),
"is_cisco_eox_api_auto_sync_enabled": app_config.is_cisco_eox_api_auto_sync_enabled()
}
return render(request, "config/change_configuration.html", context=context)
def test_app_settings():
app_settings = AppSettings()
assert app_settings
def execute_task_to_synchronize_cisco_eox_states(
self, ignore_periodic_sync_flag=False):
"""
This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
results in the local database. There are two types of operation:
* cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
blacklist and not in the database
* cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
included in the database
:return:
"""
app_config = AppSettings()
app_config.read_file()
run_task = app_config.is_cisco_eox_api_auto_sync_enabled()
if ignore_periodic_sync_flag:
run_task = True
if run_task:
logger.info("start sync with Cisco EoX API...")
self.update_state(
state=TaskState.PROCESSING,
meta={"status_message": "sync with Cisco EoX API..."})
# read queries from configuration
queries = app_config.get_cisco_eox_api_queries().splitlines()
if len(queries) == 0:
result = {"status_message": "No Cisco EoX API queries configured."}
# update the local database with the Cisco EoX API
else:
# test Cisco EoX API access
success, _ = test_cisco_eox_api_access(
app_config.get_cisco_api_client_id(),
app_config.get_cisco_api_client_secret(), False)
if not success:
msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \
"and that the authentication settings are valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization with the Cisco EoX API was not started."
)
result = {"error_message": msg}
else:
try:
# execute query by query
notify_metrics = {"queries": {}}
counter = 0
for query in queries:
self.update_state(
state=TaskState.PROCESSING,
meta={
"status_message":
"send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
"%d</strong>)..." %
(query, counter + 1, len(queries))
})
query_results = cisco_eox_api_crawler.update_cisco_eox_database(
query)
blist_counter = 0
update_counter = 0
create_counter = 0
for qres in query_results:
if qres["created"]:
create_counter += 1
elif qres["updated"]:
update_counter += 1
elif qres["blacklist"]:
blist_counter += 1
notify_metrics["queries"][query] = {
"amount": len(query_results),
"updated_entries": update_counter,
"created_entries": create_counter,
"blacklisted_entries": blist_counter,
"result_details": query_results
}
counter += 1
# create NotificationMessage
detailed_html = ""
blist_counter = 0
update_counter = 0
create_counter = 0
for query_key in notify_metrics["queries"].keys():
update_counter += notify_metrics["queries"][query_key][
"updated_entries"]
create_counter += notify_metrics["queries"][query_key][
"created_entries"]
blist_counter += notify_metrics["queries"][query_key][
"blacklisted_entries"]
# build detailed string
detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key
cond_1 = notify_metrics["queries"][query_key][
"updated_entries"] == 0
cond_1 = cond_1 and (notify_metrics["queries"]
[query_key]["created_entries"]
== 0)
cond_1 = cond_1 and (notify_metrics["queries"]
[query_key]["blacklisted_entries"]
== 0)
if cond_1:
detailed_html += "No changes required."
else:
detailed_html += "The following products are affected by this update:</p>"
detailed_html += "<ul>"
for qres in notify_metrics["queries"][query_key][
"result_details"]:
msg = ""
if "message" in qres.keys():
if qres["message"]:
msg = qres["message"]
if qres["created"]:
detailed_html += "<li>create the Product <code>%s</code> in the database" % (
qres["PID"])
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
elif qres["updated"]:
detailed_html += "<li>update the Product data for <code>%s</code></li>" % (
qres["PID"])
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
elif qres["blacklist"]:
detailed_html += "<li>Product data for <code>%s</code> ignored</li>" % (
qres["PID"])
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
detailed_html += "</ul>"
detailed_html += "</div>"
summary_html = "The synchronization was performed successfully. "
if update_counter == 1:
summary_html += "<strong>%d</strong> product was updated, " % update_counter
else:
summary_html += "<strong>%d</strong> products are updated, " % update_counter
if create_counter == 1:
summary_html += "<strong>%s</strong> product was added to the database and " % create_counter
else:
summary_html += "<strong>%s</strong> products are added to the database and " % create_counter
if blist_counter == 1:
summary_html += "<strong>%d</strong> product was ignored." % blist_counter
else:
summary_html += "<strong>%d</strong> products are ignored." % blist_counter
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_SUCCESS,
summary_message=summary_html,
detailed_message=detailed_html)
result = {"status_message": detailed_html}
# if the task was executed eager, set state to SUCCESS (required for testing)
if self.request.is_eager:
self.update_state(
state=TaskState.SUCCESS,
meta={"status_message": detailed_html})
except CredentialsNotFoundException as ex:
msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(
ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization was performed partially.")
result = {"error_message": msg}
except CiscoApiCallFailed as ex:
msg = "Server unreachable (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization was performed partially.")
result = {"error_message": msg}
else:
result = {"status_message": "task not enabled"}
# remove in progress flag with the cache
cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")
return result
def test_get_config_value(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
settings.create_defaults()
# get a configuration value (default in the global section)
self.assertFalse(settings.get_boolean("cisco_api_enabled"))
# get a configuration value (with an explicit option)
self.assertEqual(
"PlsChgMe",
settings.get_string("client_id",
section=AppSettings.CISCO_API_SECTION))
# get the default Cisco API enabled value
self.assertFalse(settings.is_cisco_api_enabled())
# set a new value
settings.set_cisco_api_enabled(True)
# set an invalid value
with self.assertRaises(ValueError):
settings.set_cisco_api_enabled("invalid value")
# write results
settings.write_file()
# cleanup
self.clean_config_file()
def test_eox_update_call_with_special_character(self):
"""
test, that no issue exists when the '%' sign is present in the ProductID
:return:
"""
app_config = AppSettings()
app_config.read_file()
app_config.set_cisco_api_enabled(True)
app_config.set_periodic_sync_enabled(True)
app_config.write_file()
eox_db_record = """{
"EndOfServiceContractRenewal": {
"value": " ",
"dateFormat": "YYYY-MM-DD"
},
"ProductIDDescription": "^IPX 8 CDP W/E1EC TO UNIVERSAL CDP (IPX 8/16/32)",
"ProductBulletinNumber": "LEGACY_ESC_IPX_4",
"LastDateOfSupport": {
"value": "2003-07-01",
"dateFormat": "YYYY-MM-DD"
},
"EOXInputValue": "SPA* ",
"EOLProductID": "SPARE%",
"UpdatedTimeStamp": {
"value": "2015-08-23",
"dateFormat": "YYYY-MM-DD"
},
"EOXInputType": "ShowEOXByPids",
"EndOfRoutineFailureAnalysisDate": {
"value": " ",
"dateFormat": "YYYY-MM-DD"
},
"LinkToProductBulletinURL": "http://www.cisco.com/en/US/products/hw/tsd_products_support_end-of-sale_and_end-of-life_products_list.html",
"EndOfSvcAttachDate": {
"value": " ",
"dateFormat": "YYYY-MM-DD"
},
"EndOfSaleDate": {
"value": "1998-07-02",
"dateFormat": "YYYY-MM-DD"
},
"EndOfSWMaintenanceReleases": {
"value": " ",
"dateFormat": "YYYY-MM-DD"
},
"EOXExternalAnnouncementDate": {
"value": "1998-01-03",
"dateFormat": "YYYY-MM-DD"
},
"EOXMigrationDetails": {
"MigrationStrategy": " ",
"MigrationProductId": " ",
"MigrationProductInfoURL": " ",
"PIDActiveFlag": "Y ",
"MigrationProductName": "See Product Bulletin",
"MigrationInformation": " ",
"MigrationOption": "Enter Product Name(s)"
}
}"""
eox_db_json = json.loads(eox_db_record)
api_crawler.update_local_db_based_on_record(eox_db_json, True)
p = Product.objects.get(product_id="SPARE%")
def status(request):
"""
Status page for the Product Database
"""
app_config = AppSettings()
app_config.read_file()
is_cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {
"is_cisco_api_enabled": is_cisco_api_enabled
}
if is_cisco_api_enabled:
# test access (once every 30 minutes)
cisco_eox_api_test_successful = cache.get("CISCO_EOC_API_TEST", False)
# defaults, overwritten if an exception is thrown
cisco_eox_api_available = True
cisco_eox_api_message = "successful connected to the Cisco EoX API"
if not cisco_eox_api_test_successful:
try:
test_cisco_eox_api_access(client_id=app_config.get_cisco_api_client_id(),
client_secret=app_config.get_cisco_api_client_secret(),
drop_credentials=False)
cache.set("CISCO_EOC_API_TEST", True, 60 * 30)
except Exception as ex:
cisco_eox_api_available = True
cisco_eox_api_message = str(ex)
context["cisco_eox_api_available"] = cisco_eox_api_available
context["cisco_eox_api_message"] = cisco_eox_api_message
# determine worker status
ws = WorkerState.objects.all()
if ws.count() == 0:
worker_status = """
<div class="alert alert-danger" role="alert">
<span class="fa fa-exclamation-circle"></span>
<span class="sr-only">Error:</span>
All backend worker offline, asynchronous and scheduled tasks are not executed.
</div>"""
else:
alive_worker = False
for w in ws:
if w.is_alive():
alive_worker = True
break
if alive_worker:
worker_status = """
<div class="alert alert-success" role="alert">
<span class="fa fa-info-circle"></span>
<span class="sr-only">Error:</span>
Backend worker found.
</div>"""
else:
worker_status = """
<div class="alert alert-warning" role="alert">
<span class="fa fa-exclamation-circle"></span>
<span class="sr-only">Error:</span>
Only unregistered backend worker found, asynchronous and scheduled tasks are not executed.
Please verify the state in the <a href="/productdb/admin">Django Admin</a> page.
</div>"""
context['worker_status'] = mark_safe(worker_status)
return render(request, "config/status.html", context=context)
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False):
"""
This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
results in the local database. There are two types of operation:
* cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
blacklist and not in the database
* cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
included in the database
:return:
"""
app_config = AppSettings()
app_config.read_file()
run_task = app_config.is_cisco_eox_api_auto_sync_enabled()
if ignore_periodic_sync_flag:
run_task = True
if run_task:
logger.info("start sync with Cisco EoX API...")
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "sync with Cisco EoX API..."
})
# read queries from configuration
queries = app_config.get_cisco_eox_api_queries().splitlines()
if len(queries) == 0:
result = {
"status_message": "No Cisco EoX API queries configured."
}
# update the local database with the Cisco EoX API
else:
# test Cisco EoX API access
success, _ = test_cisco_eox_api_access(
app_config.get_cisco_api_client_id(),
app_config.get_cisco_api_client_secret(),
False
)
if not success:
msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \
"and that the authentication settings are valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization with the Cisco EoX API was not started."
)
result = {
"error_message": msg
}
else:
try:
# execute query by query
notify_metrics = {
"queries": {}
}
counter = 0
for query in queries:
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
"%d</strong>)..." % (query, counter + 1, len(queries))
})
query_results = cisco_eox_api_crawler.update_cisco_eox_database(query)
blist_counter = 0
update_counter = 0
create_counter = 0
for qres in query_results:
if qres["created"]:
create_counter += 1
elif qres["updated"]:
update_counter += 1
elif qres["blacklist"]:
blist_counter += 1
notify_metrics["queries"][query] = {
"amount": len(query_results),
"updated_entries": update_counter,
"created_entries": create_counter,
"blacklisted_entries": blist_counter,
"result_details": query_results
}
counter += 1
# create NotificationMessage
detailed_html = ""
blist_counter = 0
update_counter = 0
create_counter = 0
for query_key in notify_metrics["queries"].keys():
update_counter += notify_metrics["queries"][query_key]["updated_entries"]
create_counter += notify_metrics["queries"][query_key]["created_entries"]
blist_counter += notify_metrics["queries"][query_key]["blacklisted_entries"]
# build detailed string
detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key
cond_1 = notify_metrics["queries"][query_key]["updated_entries"] == 0
cond_1 = cond_1 and (notify_metrics["queries"][query_key]["created_entries"] == 0)
cond_1 = cond_1 and (notify_metrics["queries"][query_key]["blacklisted_entries"] == 0)
if cond_1:
detailed_html += "No changes required."
else:
detailed_html += "The following products are affected by this update:</p>"
detailed_html += "<ul>"
for qres in notify_metrics["queries"][query_key]["result_details"]:
msg = ""
if "message" in qres.keys():
if qres["message"]:
msg = qres["message"]
if qres["created"]:
detailed_html += "<li>create the Product <code>%s</code> in the database" % (
qres["PID"]
)
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
elif qres["updated"]:
detailed_html += "<li>update the Product data for <code>%s</code></li>" % (
qres["PID"]
)
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
elif qres["blacklist"]:
detailed_html += "<li>Product data for <code>%s</code> ignored</li>" % (
qres["PID"]
)
if msg != "":
detailed_html += "(%s)</li>" % msg
else:
detailed_html += "</li>"
detailed_html += "</ul>"
detailed_html += "</div>"
summary_html = "The synchronization was performed successfully. "
if update_counter == 1:
summary_html += "<strong>%d</strong> product was updated, " % update_counter
else:
summary_html += "<strong>%d</strong> products are updated, " % update_counter
if create_counter == 1:
summary_html += "<strong>%s</strong> product was added to the database and " % create_counter
else:
summary_html += "<strong>%s</strong> products are added to the database and " % create_counter
if blist_counter == 1:
summary_html += "<strong>%d</strong> product was ignored." % blist_counter
else:
summary_html += "<strong>%d</strong> products are ignored." % blist_counter
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_SUCCESS,
summary_message=summary_html,
detailed_message=detailed_html
)
result = {
"status_message": detailed_html
}
# if the task was executed eager, set state to SUCCESS (required for testing)
if self.request.is_eager:
self.update_state(state=TaskState.SUCCESS, meta={
"status_message": detailed_html
})
except CredentialsNotFoundException as ex:
msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization was performed partially."
)
result = {
"error_message": msg
}
except CiscoApiCallFailed as ex:
msg = "Server unreachable (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title="Synchronization with Cisco EoX API",
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization was performed partially."
)
result = {
"error_message": msg
}
else:
result = {
"status_message": "task not enabled"
}
# remove in progress flag with the cache
cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")
return result
def test_get_config_value(self):
self.clean_config_file()
# create new AppSettings object
settings = AppSettings()
settings.create_defaults()
# get a configuration value (default in the global section)
self.assertFalse(settings.get_boolean("cisco_api_enabled"))
# get a configuration value (with an explicit option)
self.assertEqual("PlsChgMe", settings.get_string("client_id", section=AppSettings.CISCO_API_SECTION))
# get the default Cisco API enabled value
self.assertFalse(settings.is_cisco_api_enabled())
# set a new value
settings.set_cisco_api_enabled(True)
# set an invalid value
with self.assertRaises(ValueError):
settings.set_cisco_api_enabled("invalid value")
# write results
settings.write_file()
# cleanup
self.clean_config_file()