def test_cisco_api_client_id(self):
settings = AppSettings()
# get values
value = settings.get_cisco_api_client_id()
assert value == "PlsChgMe"
# set values
test_client_id = "test_id"
settings.set_cisco_api_client_id(test_client_id)
value = settings.get_cisco_api_client_id()
assert value == test_client_id
def test_cisco_api_client_id(self):
settings = AppSettings()
# get values
value = settings.get_cisco_api_client_id()
assert value == "PlsChgMe"
# set values
test_client_id = "test_id"
settings.set_cisco_api_client_id(test_client_id)
value = settings.get_cisco_api_client_id()
assert value == test_client_id
def load_client_credentials(self):
logger.debug("load client credentials from configuration")
app_settings = AppSettings()
# load client credentials
self.client_id = app_settings.get_cisco_api_client_id()
self.client_secret = app_settings.get_cisco_api_client_secret()
def load_client_credentials(self):
logger.debug("load client credentials from configuration")
app_settings = AppSettings()
# load client credentials
self.client_id = app_settings.get_cisco_api_client_id()
self.client_secret = app_settings.get_cisco_api_client_secret()
def status(request):
"""
Status page for the Product Database
"""
app_config = AppSettings()
is_cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {
"is_cisco_api_enabled": is_cisco_api_enabled
}
if is_cisco_api_enabled:
# test access (once every 30 minutes)
cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)
# defaults, overwritten if an exception is thrown
cisco_eox_api_available = True
cisco_eox_api_message = "successful connected to the Cisco EoX API"
if not cisco_eox_api_test_successful:
try:
result = utils.check_cisco_eox_api_access(client_id=app_config.get_cisco_api_client_id(),
client_secret=app_config.get_cisco_api_client_secret(),
drop_credentials=False)
cache.set("CISCO_EOX_API_TEST", result, 60 * 30)
except Exception as ex:
cisco_eox_api_available = True
cisco_eox_api_message = str(ex)
context["cisco_eox_api_available"] = cisco_eox_api_available
context["cisco_eox_api_message"] = cisco_eox_api_message
# determine worker status
state = celery.is_worker_active()
if state and not settings.DEBUG:
worker_status = """
<div class="alert alert-success" role="alert">
<span class="fa fa-info-circle"></span>
Backend worker found.
</div>"""
else:
worker_status = """
<div class="alert alert-danger" role="alert">
<span class="fa fa-exclamation-circle"></span>
No backend worker found, asynchronous and scheduled tasks are not executed.
</div>"""
context['worker_status'] = mark_safe(worker_status)
return render(request, "config/status.html", context=context)
def status(request):
"""
Status page for the Product Database
"""
app_config = AppSettings()
is_cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {"is_cisco_api_enabled": is_cisco_api_enabled}
if is_cisco_api_enabled:
# test access (once every 30 minutes)
cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)
# defaults, overwritten if an exception is thrown
cisco_eox_api_available = True
cisco_eox_api_message = "successful connected to the Cisco EoX API"
if not cisco_eox_api_test_successful:
try:
result = utils.check_cisco_eox_api_access(
client_id=app_config.get_cisco_api_client_id(),
client_secret=app_config.get_cisco_api_client_secret(),
drop_credentials=False)
cache.set("CISCO_EOX_API_TEST", result, 60 * 30)
except Exception as ex:
cisco_eox_api_available = True
cisco_eox_api_message = str(ex)
context["cisco_eox_api_available"] = cisco_eox_api_available
context["cisco_eox_api_message"] = cisco_eox_api_message
# determine worker status
context['worker_status'] = mark_safe(utils.get_celery_worker_state_html())
return render(request, "config/status.html", context=context)
def execute_task_to_synchronize_cisco_eox_states(
self, ignore_periodic_sync_flag=False):
"""
This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
results in the local database. There are two types of operation:
* cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
blacklist and not in the database
* cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
included in the database
:return:
"""
app_config = AppSettings()
run_task = app_config.is_periodic_sync_enabled()
if run_task or ignore_periodic_sync_flag:
logger.info("start sync with Cisco EoX API...")
self.update_state(
state=TaskState.PROCESSING,
meta={"status_message": "sync with Cisco EoX API..."})
# read queries from configuration
queries = app_config.get_cisco_eox_api_queries_as_list()
if len(queries) == 0:
result = {"status_message": "No Cisco EoX API queries configured."}
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_WARNING,
summary_message=
"There are no Cisco EoX API queries configured. Nothing to do.",
detailed_message=
"There are no Cisco EoX API queries configured. Please configure at least on EoX API "
"query in the settings or disable the periodic synchronization."
)
# update the local database with the Cisco EoX API
else:
try:
# test Cisco EoX API access
success = utils.check_cisco_eox_api_access(
app_config.get_cisco_api_client_id(),
app_config.get_cisco_api_client_secret(), False)
if not success:
msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \
"and that the authentication settings are valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization with the Cisco EoX API was not started."
)
result = {"error_message": msg}
else:
# execute all queries from the configuration and collect the results
notify_metrics = {"queries": {}}
counter = 0
for query in queries:
self.update_state(
state=TaskState.PROCESSING,
meta={
"status_message":
"send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
"%d</strong>)..." %
(query, counter + 1, len(queries))
})
# wait a specific amount of seconds between each update call
time.sleep(
int(app_config.get_cisco_eox_api_sync_wait_time()))
query_results = cisco_eox_api_crawler.update_cisco_eox_database(
query)
blist_counter = 0
update_counter = 0
create_counter = 0
for qres in query_results:
if qres["created"]:
create_counter += 1
elif qres["updated"]:
update_counter += 1
elif qres["blacklist"]:
blist_counter += 1
notify_metrics["queries"][query] = {
"amount": len(query_results),
"updated_entries": update_counter,
"created_entries": create_counter,
"blacklisted_entries": blist_counter,
"result_details": query_results
}
counter += 1
# create NotificationMessage based on the results
detailed_html = ""
blist_counter = 0
update_counter = 0
create_counter = 0
for query_key in notify_metrics["queries"].keys():
update_counter += notify_metrics["queries"][query_key][
"updated_entries"]
create_counter += notify_metrics["queries"][query_key][
"created_entries"]
blist_counter += notify_metrics["queries"][query_key][
"blacklisted_entries"]
# build detailed string
detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key
cond_1 = notify_metrics["queries"][query_key][
"updated_entries"] == 0
cond_1 = cond_1 and (notify_metrics["queries"]
[query_key]["created_entries"]
== 0)
cond_1 = cond_1 and (notify_metrics["queries"]
[query_key]["blacklisted_entries"]
== 0)
if cond_1:
detailed_html += "No changes required."
else:
detailed_html += "The following products are affected by this update:</p>"
detailed_html += "<ul>"
for qres in notify_metrics["queries"][query_key][
"result_details"]:
msg = ""
if "message" in qres.keys():
if qres["message"]:
msg = qres["message"]
if qres["created"]:
detailed_html += "<li>create the Product <code>%s</code> in the database" % (
qres["PID"])
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
elif qres["updated"]:
detailed_html += "<li>update the Product data for <code>%s</code>" % (
qres["PID"])
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
elif qres["blacklist"]:
detailed_html += "<li>Product data for <code>%s</code> ignored" % (
qres["PID"])
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
detailed_html += "</ul>"
detailed_html += "</div>"
summary_html = "The synchronization was performed successfully. "
if update_counter == 1:
summary_html += "<strong>%d</strong> product was updated, " % update_counter
else:
summary_html += "<strong>%d</strong> products are updated, " % update_counter
if create_counter == 1:
summary_html += "<strong>%s</strong> product was added to the database and " % create_counter
else:
summary_html += "<strong>%s</strong> products are added to the database and " % create_counter
if blist_counter == 1:
summary_html += "<strong>%d</strong> product was ignored." % blist_counter
else:
summary_html += "<strong>%d</strong> products are ignored." % blist_counter
# show the executed queries in the summary message
summary_html += " The following queries were executed: %s" % ", ".join(
["<code>%s</code>" % query for query in queries])
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_SUCCESS,
summary_message=summary_html,
detailed_message=detailed_html)
result = {"status_message": detailed_html}
# if the task was executed eager, set state to SUCCESS (required for testing)
if self.request.is_eager:
self.update_state(
state=TaskState.SUCCESS,
meta={"status_message": detailed_html})
except CredentialsNotFoundException as ex:
msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(
ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization was performed partially.")
result = {"error_message": msg}
except CiscoApiCallFailed as ex:
msg = "Cisco EoX API call failed (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message=
"The synchronization was performed partially.")
result = {"error_message": msg}
except Exception as ex:
msg = "Cannot access the Cisco API. Please ensure that the server is " \
"connected to the internet and that the authentication settings are " \
"valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="%s" % str(ex))
result = {"error_message": msg}
else:
result = {"status_message": "task not enabled"}
# remove in progress flag with the cache
cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")
return result
def change_configuration(request):
"""
change configuration of the Product Database
"""
# read settings from configuration file
app_config = AppSettings()
# read settings from database
hp_content_after, _ = TextBlock.objects.get_or_create(
name=TextBlock.TB_HOMEPAGE_TEXT_AFTER_FAVORITE_ACTIONS
)
hp_content_before, _ = TextBlock.objects.get_or_create(
name=TextBlock.TB_HOMEPAGE_TEXT_BEFORE_FAVORITE_ACTIONS
)
if request.method == "POST":
# create a form instance and populate it with data from the request:
form = SettingsForm(request.POST)
if form.is_valid():
# set common settings
app_config.set_login_only_mode(form.cleaned_data["login_only_mode"])
hp_content_before.html_content = form.cleaned_data["homepage_text_before"]
hp_content_before.save()
hp_content_after.html_content = form.cleaned_data["homepage_text_after"]
hp_content_after.save()
# set the Cisco API configuration options
api_enabled = form.cleaned_data["cisco_api_enabled"]
if not api_enabled:
# api is disabled, reset values to default
app_config.set_cisco_api_enabled(api_enabled)
app_config.set_cisco_api_client_id("PlsChgMe")
app_config.set_cisco_api_client_secret("PlsChgMe")
app_config.set_periodic_sync_enabled(False)
app_config.set_auto_create_new_products(False)
app_config.set_cisco_eox_api_queries("")
app_config.set_product_blacklist_regex("")
app_config.set_cisco_eox_api_sync_wait_time("5")
else:
app_config.set_cisco_api_enabled(api_enabled)
client_id = form.cleaned_data["cisco_api_client_id"] \
if form.cleaned_data["cisco_api_client_id"] != "" else "PlsChgMe"
app_config.set_cisco_api_client_id(client_id)
client_secret = form.cleaned_data["cisco_api_client_secret"] \
if form.cleaned_data["cisco_api_client_secret"] != "" else "PlsChgMe"
app_config.set_cisco_api_client_secret(client_secret)
app_config.set_internal_product_id_label(form.cleaned_data["internal_product_id_label"])
app_config.set_periodic_sync_enabled(form.cleaned_data["eox_api_auto_sync_enabled"])
app_config.set_auto_create_new_products(form.cleaned_data["eox_auto_sync_auto_create_elements"])
app_config.set_cisco_eox_api_queries(form.cleaned_data["eox_api_queries"])
app_config.set_product_blacklist_regex(form.cleaned_data["eox_api_blacklist"])
if form.cleaned_data["eox_api_wait_time"]:
app_config.set_cisco_eox_api_sync_wait_time(form.cleaned_data["eox_api_wait_time"])
if client_id != "PlsChgMe":
result = utils.check_cisco_eox_api_access(
form.cleaned_data["cisco_api_client_id"],
form.cleaned_data["cisco_api_client_secret"]
)
if result:
messages.success(request, "Successfully connected to the Cisco EoX API")
else:
messages.error(request, "Cannot contact the Cisco EoX API. Please contact your "
"Administrator")
else:
messages.info(
request,
"Please configure your Cisco API credentials within the Cisco API settings tab."
)
# expire cached settings
cache.delete("LOGIN_ONLY_MODE_SETTING")
messages.success(request, "Settings saved successfully")
return redirect(resolve_url("productdb_config:change_settings"))
else:
messages.error(request, "Invalid configuration option detected, please check it below.")
else:
form = SettingsForm()
form.fields['cisco_api_enabled'].initial = app_config.is_cisco_api_enabled()
form.fields['login_only_mode'].initial = app_config.is_login_only_mode()
form.fields['internal_product_id_label'].initial = app_config.get_internal_product_id_label()
form.fields['cisco_api_client_id'].initial = app_config.get_cisco_api_client_id()
form.fields['cisco_api_client_secret'].initial = app_config.get_cisco_api_client_secret()
form.fields['eox_api_auto_sync_enabled'].initial = app_config.is_periodic_sync_enabled()
form.fields['eox_auto_sync_auto_create_elements'].initial = app_config.is_auto_create_new_products()
form.fields['eox_api_queries'].initial = app_config.get_cisco_eox_api_queries()
form.fields['eox_api_blacklist'].initial = app_config.get_product_blacklist_regex()
form.fields['eox_api_wait_time'].initial = app_config.get_cisco_eox_api_sync_wait_time()
form.fields['homepage_text_before'].initial = hp_content_before.html_content
form.fields['homepage_text_after'].initial = hp_content_after.html_content
context = {
"form": form,
"is_cisco_api_enabled": app_config.is_cisco_api_enabled(),
"is_cisco_eox_api_auto_sync_enabled": app_config.is_periodic_sync_enabled()
}
return render(request, "config/change_configuration.html", context=context)
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False):
"""
This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
results in the local database. There are two types of operation:
* cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
blacklist and not in the database
* cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
included in the database
:return:
"""
app_config = AppSettings()
run_task = app_config.is_periodic_sync_enabled()
if run_task or ignore_periodic_sync_flag:
logger.info("start sync with Cisco EoX API...")
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "sync with Cisco EoX API..."
})
# read queries from configuration
queries = app_config.get_cisco_eox_api_queries_as_list()
if len(queries) == 0:
result = {
"status_message": "No Cisco EoX API queries configured."
}
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_WARNING,
summary_message="There are no Cisco EoX API queries configured. Nothing to do.",
detailed_message="There are no Cisco EoX API queries configured. Please configure at least on EoX API "
"query in the settings or disable the periodic synchronization."
)
# update the local database with the Cisco EoX API
else:
try:
# test Cisco EoX API access
success = utils.check_cisco_eox_api_access(
app_config.get_cisco_api_client_id(),
app_config.get_cisco_api_client_secret(),
False
)
if not success:
msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \
"and that the authentication settings are valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization with the Cisco EoX API was not started."
)
result = {
"error_message": msg
}
else:
# execute all queries from the configuration and collect the results
notify_metrics = {
"queries": {}
}
counter = 0
for query in queries:
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
"%d</strong>)..." % (query, counter + 1, len(queries))
})
# wait a specific amount of seconds between each update call
time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time()))
query_results = cisco_eox_api_crawler.update_cisco_eox_database(query)
blist_counter = 0
update_counter = 0
create_counter = 0
for qres in query_results:
if qres["created"]:
create_counter += 1
elif qres["updated"]:
update_counter += 1
elif qres["blacklist"]:
blist_counter += 1
notify_metrics["queries"][query] = {
"amount": len(query_results),
"updated_entries": update_counter,
"created_entries": create_counter,
"blacklisted_entries": blist_counter,
"result_details": query_results
}
counter += 1
# create NotificationMessage based on the results
detailed_html = ""
blist_counter = 0
update_counter = 0
create_counter = 0
for query_key in notify_metrics["queries"].keys():
update_counter += notify_metrics["queries"][query_key]["updated_entries"]
create_counter += notify_metrics["queries"][query_key]["created_entries"]
blist_counter += notify_metrics["queries"][query_key]["blacklisted_entries"]
# build detailed string
detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key
cond_1 = notify_metrics["queries"][query_key]["updated_entries"] == 0
cond_1 = cond_1 and (notify_metrics["queries"][query_key]["created_entries"] == 0)
cond_1 = cond_1 and (notify_metrics["queries"][query_key]["blacklisted_entries"] == 0)
if cond_1:
detailed_html += "No changes required."
else:
detailed_html += "The following products are affected by this update:</p>"
detailed_html += "<ul>"
for qres in notify_metrics["queries"][query_key]["result_details"]:
msg = ""
if "message" in qres.keys():
if qres["message"]:
msg = qres["message"]
if qres["created"]:
detailed_html += "<li>create the Product <code>%s</code> in the database" % (
qres["PID"]
)
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
elif qres["updated"]:
detailed_html += "<li>update the Product data for <code>%s</code>" % (
qres["PID"]
)
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
elif qres["blacklist"]:
detailed_html += "<li>Product data for <code>%s</code> ignored" % (
qres["PID"]
)
detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"
detailed_html += "</ul>"
detailed_html += "</div>"
summary_html = "The synchronization was performed successfully. "
if update_counter == 1:
summary_html += "<strong>%d</strong> product was updated, " % update_counter
else:
summary_html += "<strong>%d</strong> products are updated, " % update_counter
if create_counter == 1:
summary_html += "<strong>%s</strong> product was added to the database and " % create_counter
else:
summary_html += "<strong>%s</strong> products are added to the database and " % create_counter
if blist_counter == 1:
summary_html += "<strong>%d</strong> product was ignored." % blist_counter
else:
summary_html += "<strong>%d</strong> products are ignored." % blist_counter
# show the executed queries in the summary message
summary_html += " The following queries were executed: %s" % ", ".join(
["<code>%s</code>" % query for query in queries]
)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_SUCCESS,
summary_message=summary_html,
detailed_message=detailed_html
)
result = {
"status_message": detailed_html
}
# if the task was executed eager, set state to SUCCESS (required for testing)
if self.request.is_eager:
self.update_state(state=TaskState.SUCCESS, meta={
"status_message": detailed_html
})
except CredentialsNotFoundException as ex:
msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization was performed partially."
)
result = {
"error_message": msg
}
except CiscoApiCallFailed as ex:
msg = "Cisco EoX API call failed (%s)" % str(ex)
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="The synchronization was performed partially."
)
result = {
"error_message": msg
}
except Exception as ex:
msg = "Cannot access the Cisco API. Please ensure that the server is " \
"connected to the internet and that the authentication settings are " \
"valid."
logger.error(msg, exc_info=True)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_ERROR,
summary_message=msg,
detailed_message="%s" % str(ex)
)
result = {
"error_message": msg
}
else:
result = {
"status_message": "task not enabled"
}
# remove in progress flag with the cache
cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")
return result
def initial_sync_with_cisco_eox_api(self, years_list):
"""
synchronize all entries from the EoX API for a given amount of years (today - n-years), ignores the create missing
entries and the configurable blacklist.
:param self:
:param years_list: list of years to sync (e.g. [2018, 2017, 2016]
:return:
"""
if type(years_list) is not list:
raise AttributeError("years_list must be a list")
for val in years_list:
if type(val) is not int:
raise AttributeError("years_list must be a list of integers")
if len(years_list) == 0:
return {
"status_message": "No years provided, nothing to do."
}
app_config = AppSettings()
# test Cisco EoX API access
test_result = utils.check_cisco_eox_api_access(
app_config.get_cisco_api_client_id(),
app_config.get_cisco_api_client_secret(),
False
)
failed_years = []
successful_years = []
if test_result:
# perform synchronization
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "start initial synchronization with the Cisco EoX API..."
})
all_records = []
for year in years_list:
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "fetch all information for year %d..." % year
})
# wait some time between the query calls
time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time()))
# fetch all API entries for a specific year
try:
records = cisco_eox_api_crawler.get_raw_api_data(year=year)
successful_years += [year]
all_records.append({
"year": year,
"records": records
})
except CiscoApiCallFailed as ex:
msg = "Cisco EoX API call failed (%s)" % str(ex)
logger.error("Query for year %s to Cisco EoX API failed (%s)" % (year, msg), exc_info=True)
failed_years += [year]
NotificationMessage.objects.create(
title="Initial data import failed",
summary_message="Unable to collect Cisco EoX data for year %d" % year,
detailed_message=msg,
type=NotificationMessage.MESSAGE_ERROR
)
except Exception as ex:
msg = "Unexpected Exception, cannot access the Cisco API. Please ensure that the server is " \
"connected to the internet and that the authentication settings are valid."
logger.error("Query for year %s to Cisco EoX API failed (%s)" % (year, msg), exc_info=True)
failed_years += [year]
NotificationMessage.objects.create(
title="Initial data import failed",
summary_message="Unable to collect Cisco EoX data for year %d" % year,
detailed_message=msg,
type=NotificationMessage.MESSAGE_ERROR
)
# update local database (asynchronous task)
if len(all_records) != 0:
tasks = [
update_local_database_records.s({}, all_records[0]["year"], all_records[0]["records"])
]
for r in all_records[1:]:
tasks.append(update_local_database_records.s(r["year"], r["records"]))
tasks.append(notify_initial_import_result.s())
chain(*tasks).apply_async()
time.sleep(10)
# remove in progress flag with the cache
cache.delete("CISCO_EOX_INITIAL_SYN_IN_PROGRESS")
success_msg = ",".join([str(e) for e in successful_years])
if len(success_msg) == 0:
success_msg = "None"
failed_msg = ""
if len(failed_years) != 0:
failed_msg = " (for %s the synchronization failed)" % ",".join([str(e) for e in failed_years])
return {
"status_message": "The EoX data were successfully downloaded for the following years: %s%s" % (success_msg,
failed_msg)
}
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False):
"""
This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
results in the local database. There are two types of operation:
* cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
blacklist and not in the database
* cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
included in the database
:return:
"""
app_config = AppSettings()
run_task = app_config.is_periodic_sync_enabled()
if not (run_task or ignore_periodic_sync_flag):
result = {
"status_message": "task not enabled"
}
else:
logger.info("start sync with Cisco EoX API...")
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "sync with Cisco EoX API..."
})
# read configuration for the Cisco EoX API synchronization
queries = app_config.get_cisco_eox_api_queries_as_list()
if len(queries) == 0:
result = {
"status_message": "No Cisco EoX API queries configured."
}
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE,
type=NotificationMessage.MESSAGE_WARNING,
summary_message="There are no Cisco EoX API queries configured. Nothing to do.",
detailed_message="There are no Cisco EoX API queries configured. Please configure at least on EoX API "
"query in the settings or disable the periodic synchronization."
)
# update the local database with the Cisco EoX API
else:
# test Cisco EoX API access
test_result = utils.check_cisco_eox_api_access(
app_config.get_cisco_api_client_id(),
app_config.get_cisco_api_client_secret(),
False
)
if not test_result:
msg = "Cannot contact Cisco EoX API, please verify your internet connection and access " \
"credentials."
if not ignore_periodic_sync_flag:
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR,
summary_message="The synchronization with the Cisco EoX API was not successful.",
detailed_message=msg
)
result = {
"error_message": msg
}
else:
# execute all queries from the configuration
query_eox_records = {}
failed_queries = []
failed_query_msgs = {}
successful_queries = []
counter = 1
for query in queries:
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
"%d</strong>)..." % (query, counter, len(queries))
})
# wait some time between the query calls
time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time()))
try:
query_eox_records[query] = cisco_eox_api_crawler.get_raw_api_data(api_query=query)
successful_queries.append(query)
except CiscoApiCallFailed as ex:
msg = "Cisco EoX API call failed (%s)" % str(ex)
logger.error("Query %s to Cisco EoX API failed (%s)" % (query, msg), exc_info=True)
failed_queries.append(query)
failed_query_msgs[query] = str(ex)
except Exception as ex:
msg = "Unexpected Exception, cannot access the Cisco API. Please ensure that the server is " \
"connected to the internet and that the authentication settings are " \
"valid."
logger.error("Query %s to Cisco EoX API failed (%s)" % (query, msg), exc_info=True)
failed_queries.append(query)
failed_query_msgs[query] = str(ex)
counter += 1
for key in query_eox_records:
amount_of_records = len(query_eox_records[key])
self.update_state(state=TaskState.PROCESSING, meta={
"status_message": "update database (query <code>%s</code>, processed <b>0</b> of "
"<b>%d</b> results)..." % (key, amount_of_records)
})
# update database in a separate task
update_cisco_eox_records.apply_async(kwargs={
"records": query_eox_records[key]
})
# view the queries in the detailed message and all messages (if there are some)
detailed_message = "The following queries were executed:<br><ul style=\"text-align: left;\">"
for fq in failed_queries:
detailed_message += "<li class=\"text-danger\"><code>%s</code> " \
"(failed, %s)</li>" % (fq, failed_query_msgs.get(fq, "unknown"))
for sq in successful_queries:
detailed_message += "<li><code>%s</code> (<b>affects %d products</b>, " \
"success)</li>" % (sq, len(query_eox_records[sq]))
detailed_message += "</ul>"
# show the executed queries in the summary message
if len(failed_queries) == 0 and len(successful_queries) != 0:
summary_html = "The following queries were successful executed: %s" % ", ".join(
["<code>%s</code>" % query for query in successful_queries]
)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_SUCCESS,
summary_message="The synchronization with the Cisco EoX API was successful. " + summary_html,
detailed_message=detailed_message
)
elif len(failed_queries) != 0 and len(successful_queries) == 0:
summary_html = "The following queries failed to execute: %s" % ", ".join(
["<code>%s</code>" % query for query in failed_queries]
)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR,
summary_message="The synchronization with the Cisco EoX API was not successful. " + summary_html,
detailed_message=detailed_message
)
else:
summary_html = "The following queries were successful executed: %s\n<br>The following queries " \
"failed to execute: %s" % (
", ".join(["<code>%s</code>" % query for query in successful_queries]),
", ".join(["<code>%s</code>" % query for query in failed_queries])
)
NotificationMessage.objects.create(
title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_WARNING,
summary_message="The synchronization with the Cisco EoX API was partially "
"successful. " + summary_html,
detailed_message=detailed_message
)
result = {"status_message": "<p style=\"text-align: left;\">" + detailed_message + "</p>"}
# if the task was executed eager, set state to SUCCESS (required for testing)
if self.request.is_eager:
self.update_state(state=TaskState.SUCCESS, meta={"status_message": summary_html})
# remove in progress flag with the cache
cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")
return result
def status(request):
"""
Status page for the Product Database
"""
app_config = AppSettings()
is_cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {
"is_cisco_api_enabled": is_cisco_api_enabled
}
if is_cisco_api_enabled:
# test access (once every 30 minutes)
cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)
# defaults, overwritten if an exception is thrown
cisco_eox_api_available = True
cisco_eox_api_message = "successful connected to the Cisco EoX API"
if not cisco_eox_api_test_successful:
try:
result = utils.check_cisco_eox_api_access(client_id=app_config.get_cisco_api_client_id(),
client_secret=app_config.get_cisco_api_client_secret(),
drop_credentials=False)
cache.set("CISCO_EOX_API_TEST", result, 60 * 30)
except Exception as ex:
cisco_eox_api_available = True
cisco_eox_api_message = str(ex)
context["cisco_eox_api_available"] = cisco_eox_api_available
context["cisco_eox_api_message"] = cisco_eox_api_message
# determine worker status
ws = WorkerState.objects.all()
if ws.count() == 0:
worker_status = """
<div class="alert alert-danger" role="alert">
<span class="fa fa-exclamation-circle"></span>
<span class="sr-only">Error:</span>
All backend worker offline, asynchronous and scheduled tasks are not executed.
</div>"""
else:
alive_worker = False
for w in ws:
if w.is_alive():
alive_worker = True
break
if alive_worker:
worker_status = """
<div class="alert alert-success" role="alert">
<span class="fa fa-info-circle"></span>
<span class="sr-only">Error:</span>
Backend worker found.
</div>"""
else:
worker_status = """
<div class="alert alert-warning" role="alert">
<span class="fa fa-exclamation-circle"></span>
<span class="sr-only">Error:</span>
Only unregistered backend worker found, asynchronous and scheduled tasks are not executed.
Please verify the state in the <a href="/productdb/admin">Django Admin</a> page.
</div>"""
context['worker_status'] = mark_safe(worker_status)
return render(request, "config/status.html", context=context)
def change_configuration(request):
"""
change configuration of the Product Database
"""
# read settings from configuration file
app_config = AppSettings()
# read settings from database
hp_content_after, _ = TextBlock.objects.get_or_create(
name=TextBlock.TB_HOMEPAGE_TEXT_AFTER_FAVORITE_ACTIONS
)
hp_content_before, _ = TextBlock.objects.get_or_create(
name=TextBlock.TB_HOMEPAGE_TEXT_BEFORE_FAVORITE_ACTIONS
)
if request.method == "POST":
# create a form instance and populate it with data from the request:
form = SettingsForm(request.POST)
if form.is_valid():
# set common settings
app_config.set_login_only_mode(form.cleaned_data["login_only_mode"])
hp_content_before.html_content = form.cleaned_data["homepage_text_before"]
hp_content_before.save()
hp_content_after.html_content = form.cleaned_data["homepage_text_after"]
hp_content_after.save()
# set the Cisco API configuration options
api_enabled = form.cleaned_data["cisco_api_enabled"]
if not api_enabled:
# api is disabled, reset values to default
app_config.set_cisco_api_enabled(api_enabled)
app_config.set_cisco_api_client_id("PlsChgMe")
app_config.set_cisco_api_client_secret("PlsChgMe")
app_config.set_periodic_sync_enabled(False)
app_config.set_auto_create_new_products(False)
app_config.set_cisco_eox_api_queries("")
app_config.set_product_blacklist_regex("")
app_config.set_cisco_eox_api_sync_wait_time("5")
else:
app_config.set_cisco_api_enabled(api_enabled)
client_id = form.cleaned_data["cisco_api_client_id"] \
if form.cleaned_data["cisco_api_client_id"] != "" else "PlsChgMe"
app_config.set_cisco_api_client_id(client_id)
client_secret = form.cleaned_data["cisco_api_client_secret"] \
if form.cleaned_data["cisco_api_client_secret"] != "" else "PlsChgMe"
app_config.set_cisco_api_client_secret(client_secret)
if client_id != "PlsChgMe":
result = utils.check_cisco_eox_api_access(
form.cleaned_data["cisco_api_client_id"],
form.cleaned_data["cisco_api_client_secret"]
)
if result:
messages.success(request, "Successfully connected to the Cisco EoX API")
else:
messages.error(request, "Cannot contact the Cisco EoX API. Please contact your "
"Administrator")
else:
messages.info(
request,
"Please configure your Cisco API credentials within the Cisco API settings tab."
)
app_config.set_periodic_sync_enabled(form.cleaned_data["eox_api_auto_sync_enabled"])
app_config.set_internal_product_id_label(form.cleaned_data["internal_product_id_label"])
app_config.set_auto_create_new_products(form.cleaned_data["eox_auto_sync_auto_create_elements"])
app_config.set_cisco_eox_api_queries(form.cleaned_data["eox_api_queries"])
app_config.set_product_blacklist_regex(form.cleaned_data["eox_api_blacklist"])
if form.cleaned_data["eox_api_wait_time"]:
app_config.set_cisco_eox_api_sync_wait_time(form.cleaned_data["eox_api_wait_time"])
# expire cached settings
cache.delete("LOGIN_ONLY_MODE_SETTING")
messages.success(request, "Settings saved successfully")
return redirect(resolve_url("productdb_config:change_settings"))
else:
messages.error(request, "Invalid configuration option detected, please check it below.")
else:
form = SettingsForm()
form.fields['cisco_api_enabled'].initial = app_config.is_cisco_api_enabled()
form.fields['login_only_mode'].initial = app_config.is_login_only_mode()
form.fields['internal_product_id_label'].initial = app_config.get_internal_product_id_label()
form.fields['cisco_api_client_id'].initial = app_config.get_cisco_api_client_id()
form.fields['cisco_api_client_secret'].initial = app_config.get_cisco_api_client_secret()
form.fields['eox_api_auto_sync_enabled'].initial = app_config.is_periodic_sync_enabled()
form.fields['eox_auto_sync_auto_create_elements'].initial = app_config.is_auto_create_new_products()
form.fields['eox_api_queries'].initial = app_config.get_cisco_eox_api_queries()
form.fields['eox_api_blacklist'].initial = app_config.get_product_blacklist_regex()
form.fields['eox_api_wait_time'].initial = app_config.get_cisco_eox_api_sync_wait_time()
form.fields['homepage_text_before'].initial = hp_content_before.html_content
form.fields['homepage_text_after'].initial = hp_content_after.html_content
context = {
"form": form,
"is_cisco_api_enabled": app_config.is_cisco_api_enabled(),
"is_cisco_eox_api_auto_sync_enabled": app_config.is_periodic_sync_enabled()
}
return render(request, "config/change_configuration.html", context=context)