def load_client_credentials(self):
logger.debug("load client credentials from configuration")
app_settings = AppSettings()
# load client credentials
self.client_id = app_settings.get_cisco_api_client_id()
self.client_secret = app_settings.get_cisco_api_client_secret()
def load_test_cisco_api_credentials():
with open(CISCO_API_TEST_CREDENTIALS_FILE) as f:
cred = json.loads(f.read())
app = AppSettings()
app.set_cisco_api_client_id(cred["client_id"])
app.set_cisco_api_client_secret(cred["client_secret"])
def load_client_credentials(self):
logger.debug("load client credentials from configuration")
app_settings = AppSettings()
# load client credentials
self.client_id = app_settings.get_cisco_api_client_id()
self.client_secret = app_settings.get_cisco_api_client_secret()
def get_internal_product_id_label(request):
app_config = AppSettings()
return {
"INTERNAL_PRODUCT_ID_LABEL": app_config.get_internal_product_id_label(),
"STAT_AMOUNT_OF_PRODUCT_CHECKS": app_config.get_amount_of_product_checks(),
"STAT_AMOUNT_OF_UNIQUE_PRODUCT_CHECK_ENTRIES": app_config.get_amount_of_unique_product_check_entries(),
}
def handle(self, *args, **kwargs):
app_settings = AppSettings()
if not app_settings.is_cisco_api_enabled():
raise CommandError(
"Please configure the Cisco EoX API in the settings prior running this task"
)
# check if task is already running, if so print status and continue
task_id = cache.get("CISCO_EOX_INITIAL_SYN_IN_PROGRESS", None)
if task_id is None:
# start initial import
eta = now() + timedelta(seconds=3)
task = tasks.initial_sync_with_cisco_eox_api.apply_async(
eta=eta, args=(kwargs["years"], ))
cache.set("CISCO_EOX_INITIAL_SYN_IN_PROGRESS", task.id,
60 * 60 * 48)
cache.set("CISCO_EOX_INITIAL_SYN_LAST_RUN", task.id, 60 * 60 * 48)
self.stdout.write("Task successful started...")
else:
msg = get_task_state_message(task_id)
raise CommandError("initial import already running... \n%s" % msg)
def get_internal_product_id_label(request):
app_config = AppSettings()
return {
"INTERNAL_PRODUCT_ID_LABEL":
app_config.get_internal_product_id_label(),
"STAT_AMOUNT_OF_PRODUCT_CHECKS":
app_config.get_amount_of_product_checks(),
"STAT_AMOUNT_OF_UNIQUE_PRODUCT_CHECK_ENTRIES":
app_config.get_amount_of_unique_product_check_entries()
}
def get_raw_api_data(api_query):
"""
returns all EoX records for a specific query (from all pages)
:param api_query: single query that is send to the Cisco EoX API
:raises CiscoApiCallFailed: exception raised if Cisco EoX API call failed
:return: list that contains all EoX records from the Cisco EoX API
"""
if type(api_query) is not str:
raise ValueError("api_query must be a string value")
# load application settings and check, that the API is enabled
app_settings = AppSettings()
if not app_settings.is_cisco_api_enabled():
msg = "Cisco API access not enabled"
logger.warning(msg)
raise CiscoApiCallFailed(msg)
# start Cisco EoX API query
logger.info("send query to Cisco EoX database: %s" % api_query)
eoxapi = CiscoEoxApi()
eoxapi.load_client_credentials()
results = []
try:
current_page = 1
result_pages = 999
while current_page <= result_pages:
if current_page == 1:
logger.info("Executing API query '%s' on first page" % api_query)
else:
logger.info("Executing API query '%s' on page '%d" % (api_query, current_page))
# will raise a CiscoApiCallFailed exception on error
eoxapi.query_product(product_id=api_query, page=current_page)
result_pages = eoxapi.amount_of_pages()
if eoxapi.get_page_record_count() > 0:
results.extend(eoxapi.get_eox_records())
current_page += 1
except ConnectionFailedException:
logger.error("Query failed, server not reachable: %s" % api_query, exc_info=True)
raise
except CiscoApiCallFailed:
logger.fatal("Query failed: %s" % api_query, exc_info=True)
raise
return results
def test_write_datetime_to_settings_file(self):
settings = AppSettings()
# get a configuration value (default in the global section)
now = datetime.now()
value = now.isoformat()
settings.set_cisco_eox_api_auto_sync_last_execution_time(value)
read_value = settings.get_cisco_eox_api_auto_sync_last_execution_time()
assert value == read_value
assert now == parse_datetime(read_value)
def test_write_datetime_to_settings_file(self):
settings = AppSettings()
# get a configuration value (default in the global section)
now = datetime.now()
value = now.isoformat()
settings.set_cisco_eox_api_auto_sync_last_execution_time(value)
read_value = settings.get_cisco_eox_api_auto_sync_last_execution_time()
assert value == read_value
assert now == parse_datetime(read_value)
def test_auto_create_new_products(self):
settings = AppSettings()
# get values
value = settings.is_auto_create_new_products()
assert value is False
# set values
settings.set_auto_create_new_products(True)
value = settings.is_auto_create_new_products()
assert value is True
def status(request):
"""
Status page for the Product Database
"""
app_config = AppSettings()
is_cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {
"is_cisco_api_enabled": is_cisco_api_enabled
}
if is_cisco_api_enabled:
# test access (once every 30 minutes)
cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)
# defaults, overwritten if an exception is thrown
cisco_eox_api_available = True
cisco_eox_api_message = "successful connected to the Cisco EoX API"
if not cisco_eox_api_test_successful:
try:
result = utils.check_cisco_eox_api_access(client_id=app_config.get_cisco_api_client_id(),
client_secret=app_config.get_cisco_api_client_secret(),
drop_credentials=False)
cache.set("CISCO_EOX_API_TEST", result, 60 * 30)
except Exception as ex:
cisco_eox_api_available = True
cisco_eox_api_message = str(ex)
context["cisco_eox_api_available"] = cisco_eox_api_available
context["cisco_eox_api_message"] = cisco_eox_api_message
# determine worker status
state = celery.is_worker_active()
if state and not settings.DEBUG:
worker_status = """
<div class="alert alert-success" role="alert">
<span class="fa fa-info-circle"></span>
Backend worker found.
</div>"""
else:
worker_status = """
<div class="alert alert-danger" role="alert">
<span class="fa fa-exclamation-circle"></span>
No backend worker found, asynchronous and scheduled tasks are not executed.
</div>"""
context['worker_status'] = mark_safe(worker_status)
return render(request, "config/status.html", context=context)
def cisco_eox_populate_product_lc_state_sync_field():
"""
Periodic job to populate the lc_state_sync field in the Products, which shows that the product lifecycle data are
automatically synchronized against the Cisco EoX API in this case
:return:
"""
try:
cis_vendor = Vendor.objects.get(name__istartswith="Cisco")
except:
# Vendor doesn't exist, no steps required
logger.fatal(
"Vendor \"Cisco Systems\" not found in database, please check your installation"
)
return {"error": "Vendor \"Cisco Systems\" not found in database"}
cisco_products = Product.objects.filter(vendor=cis_vendor)
if cisco_products.count() != 0:
app_config = AppSettings()
queries = app_config.get_cisco_eox_api_queries_as_list()
# escape the query strings
queries = [re.escape(e) for e in queries]
# convert the wildcard values
queries = [e.replace("\\*", ".*") for e in queries]
queries = ["^" + e + "$" for e in queries]
with transaction.atomic():
# reset all entries for the vendor
pl = Product.objects.filter(vendor=cis_vendor)
for p in pl:
p.lc_state_sync = False
p.save()
# only set the state sync to true if the periodic synchronization is enabled
if app_config.is_periodic_sync_enabled():
for query in queries:
pl = Product.objects.filter(product_id__regex=query,
vendor=cis_vendor)
for p in pl:
p.lc_state_sync = True
p.save()
return {"status": "Database updated"}
else:
return {
"error":
"No Products associated to \"Cisco Systems\" found in database"
}
def test_cisco_eox_wait_time(self):
settings = AppSettings()
# get value
value = settings.get_cisco_eox_api_sync_wait_time()
assert value == "5"
# set values
test_cisco_eox_wait_time = "10"
settings.set_cisco_eox_api_sync_wait_time(test_cisco_eox_wait_time)
value = settings.get_cisco_eox_api_sync_wait_time()
assert value == test_cisco_eox_wait_time
def test_internal_product_id_label(self):
settings = AppSettings()
# get values
value = settings.get_internal_product_id_label()
assert value == "Internal Product ID"
# set values
test_internal_product_label = "My custom label"
settings.set_internal_product_id_label(test_internal_product_label)
value = settings.get_internal_product_id_label()
assert value == test_internal_product_label
def test_cisco_eox_api_auto_sync_last_execution_result(self):
settings = AppSettings()
# get values
value = settings.get_cisco_eox_api_auto_sync_last_execution_result()
assert value is None
# set values
test_last_execution_result = "everything is fine"
settings.set_cisco_eox_api_auto_sync_last_execution_result(test_last_execution_result)
value = settings.get_cisco_eox_api_auto_sync_last_execution_result()
assert value == test_last_execution_result
def test_cisco_api_client_id(self):
settings = AppSettings()
# get values
value = settings.get_cisco_api_client_id()
assert value == "PlsChgMe"
# set values
test_client_id = "test_id"
settings.set_cisco_api_client_id(test_client_id)
value = settings.get_cisco_api_client_id()
assert value == test_client_id
def test_product_blacklist_regex_from_config(self):
settings = AppSettings()
# get values
value = settings.get_product_blacklist_regex()
assert value == ""
# set values
blacklist_entries = "test\nanother\ntest"
settings.set_product_blacklist_regex(blacklist_entries)
value = settings.get_product_blacklist_regex()
assert value == blacklist_entries
def test_cisco_api_client_secret(self):
settings = AppSettings()
# get values
value = settings.get_cisco_api_client_secret()
assert value == "PlsChgMe"
# set values
test_client_secret = "test_secret"
settings.set_cisco_api_client_secret(test_client_secret)
value = settings.get_cisco_api_client_secret()
assert value == test_client_secret
def cisco_eox_query(request):
"""Manual query page against the Cisco EoX Version 5 API (if enabled)
:param request:
:return:
"""
app_config = AppSettings()
cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {
"is_cisco_api_enabled": cisco_api_enabled
}
if request.method == "POST":
# create a form instance and populate it with data from the request:
if "sync_cisco_eox_states_now" in request.POST.keys():
if "sync_cisco_eox_states_query" in request.POST.keys():
query = request.POST['sync_cisco_eox_states_query']
if query != "":
if len(query.split(" ")) == 1:
context['query_executed'] = query
try:
eox_api_update_records = api_crawler.update_cisco_eox_database(api_query=query)
except ConnectionFailedException as ex:
eox_api_update_records = {"error": "Cannot contact Cisco API, error message:\n%s" % ex}
except CiscoApiCallFailed as ex:
eox_api_update_records = {"error": "Cisco API call failed: %s" % ex}
except Exception as ex: # catch any exception
logger.debug("execution failed due to unexpected exception", exc_info=True)
eox_api_update_records = ["execution failed: %s" % ex]
context['eox_api_update_records'] = json.dumps(eox_api_update_records, indent=4, sort_keys=True)
else:
context['eox_api_update_records'] = "Invalid query '%s': not executed" % \
request.POST['sync_cisco_eox_states_query']
else:
context['eox_api_update_records'] = ["Please specify a valid query"]
else:
context['eox_api_update_records'] = "Query not specified."
else:
context['eox_api_update_records'] = "Query not executed, please select the \"execute it now\" checkbox."
return render(request, "ciscoeox/cisco_eox_query.html", context=context)
def test_manual_task_with_multiple_blacklist_entries(self, monkeypatch):
self.mock_api_call(monkeypatch)
# try to execute it, while no auto-sync is enabled
app = AppSettings()
app.set_periodic_sync_enabled(False)
app.set_cisco_eox_api_queries("WS-C2960-*")
app.set_product_blacklist_regex("WS-C2950G-48-EI-WS;WS-C2950G-24-EI")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(
ignore_periodic_sync_flag=True)
expected_result = [
'<p style="text-align: left;">The following queries were executed:<br>'
'<ul style="text-align: left;"><li><code>WS-C2960-*</code> (<b>affects 3 products</b>, '
'success)</li></ul>'
]
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
for er in expected_result:
assert er in task.info.get("status_message")
assert NotificationMessage.objects.count(
) == 1, "Task should create a Notification Message"
assert Product.objects.count(
) == 1, "Only a single product is imported"
nm = NotificationMessage.objects.first()
assert nm.type == NotificationMessage.MESSAGE_SUCCESS, "Incomplete configuration, should throw a warning "
def test_periodic_sync_enabled(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_periodic_sync_enabled() is False
settings.set_periodic_sync_enabled(True)
assert settings.is_periodic_sync_enabled() is True
settings.set_periodic_sync_enabled(False)
assert settings.is_periodic_sync_enabled() is False
def test_cisco_api_enabled(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_cisco_api_enabled() is False
settings.set_cisco_api_enabled(True)
assert settings.is_cisco_api_enabled() is True
settings.set_cisco_api_enabled(False)
assert settings.is_cisco_api_enabled() is False
def test_login_only_mode_configuration(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_login_only_mode() is False
settings.set_login_only_mode(True)
assert settings.is_login_only_mode() is True
settings.set_login_only_mode(False)
assert settings.is_login_only_mode() is False
def update_cisco_eox_records(records):
"""
update given database records from the Cisco EoX v5 API
:param records:
:return:
"""
app_config = AppSettings()
blacklist_raw_string = app_config.get_product_blacklist_regex()
create_missing = app_config.is_auto_create_new_products()
# build blacklist from configuration
blacklist = []
for e in [e.split(";") for e in blacklist_raw_string.splitlines()]:
blacklist += e
blacklist = [e for e in blacklist if e != ""]
counter = 0
messages = {}
for record in records:
blacklisted = False
for regex in blacklist:
try:
if re.search(regex, record["EOLProductID"], re.I):
blacklisted = True
break
except:
logger.warning("invalid regular expression in blacklist: %s" % regex)
if not blacklisted:
try:
message = cisco_eox_api_crawler.update_local_db_based_on_record(record, create_missing)
if message:
messages[record["EOLProductID"]] = message
except ValidationError as ex:
logger.error("invalid data received from Cisco API, cannot save data object for "
"'%s' (%s)" % (record, str(ex)), exc_info=True)
else:
messages[record["EOLProductID"]] = " Product record ignored"
counter += 1
return {
"count": counter,
"messages": messages
}
def test_periodic_task_without_queries(self, monkeypatch):
self.mock_api_call(monkeypatch)
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == "No Cisco EoX API queries configured."
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"
def test_global_options_are_visible(self):
app_config = AppSettings()
test_internal_id = "My custom Internal ID"
app_config.set_internal_product_id_label(test_internal_id)
# require super user permissions
user = mixer.blend("auth.User", is_superuser=True)
url = reverse(self.URL_NAME)
request = RequestFactory().get(url)
request.user = user
patch_contrib_messages(request)
response = views.change_configuration(request)
assert response.status_code == 200
assert test_internal_id in response.content.decode()
def test_global_options_are_visible(self):
app_config = AppSettings()
test_internal_id = "My custom Internal ID"
app_config.set_internal_product_id_label(test_internal_id)
# require super user permissions
user = mixer.blend("auth.User", is_superuser=True)
url = reverse(self.URL_NAME)
request = RequestFactory().get(url)
request.user = user
patch_contrib_messages(request)
response = views.change_configuration(request)
assert response.status_code == 200
assert test_internal_id in response.content.decode()
def perform_product_check(self):
"""perform the product check and populate the ProductCheckEntries"""
unique_products = [line.strip() for line in set(self.input_product_ids_list) if line.strip() != ""]
amounts = Counter(self.input_product_ids_list)
# clean all entries
self.productcheckentry_set.all().delete()
for input_product_id in unique_products:
product_entry, _ = ProductCheckEntry.objects.get_or_create(
input_product_id=input_product_id,
product_check=self
)
product_entry.amount = amounts[input_product_id]
product_entry.discover_product_list_values()
product_entry.save()
# increments statistics
settings = AppSettings()
settings.set_amount_of_product_checks(settings.get_amount_of_product_checks() + 1)
settings.set_amount_of_unique_product_check_entries(settings.get_amount_of_unique_product_check_entries() +
len(unique_products))
self.save()
def cisco_eox_populate_product_lc_state_sync_field():
"""
Periodic job to populate the lc_state_sync field in the Products, which shows that the product lifecycle data are
automatically synchronized against the Cisco EoX API in this case
:return:
"""
try:
cis_vendor = Vendor.objects.get(name__istartswith="Cisco")
except:
# Vendor doesn't exist, no steps required
return {"error": "Vendor \"Cisco Systems\" not found in database"}
cisco_products = Product.objects.filter(vendor=cis_vendor)
if cisco_products.count() != 0:
app_config = AppSettings()
queries = app_config.get_cisco_eox_api_queries_as_list()
# escape the query strings
queries = [re.escape(e) for e in queries]
# convert the wildcard values
queries = [e.replace("\\*", ".*") for e in queries]
queries = ["^" + e + "$" for e in queries]
with transaction.atomic():
# reset all entries for the vendor
pl = Product.objects.filter(vendor=cis_vendor)
for p in pl:
p.lc_state_sync = False
p.save()
# only set the state sync to true if the periodic synchronization is enabled
if app_config.is_periodic_sync_enabled():
for query in queries:
pl = Product.objects.filter(product_id__regex=query, vendor=cis_vendor)
for p in pl:
p.lc_state_sync = True
p.save()
return {"status": "Database updated"}
else:
return {"error": "No Products associated to \"Cisco Systems\" found in database"}
def test_trigger_manual_cisco_eox_synchronization():
app_config = AppSettings()
app_config.set_cisco_api_enabled(True)
app_config.set_periodic_sync_enabled(True)
# schedule Cisco EoX API update
url = reverse('cisco_api:start_cisco_eox_api_sync_now')
client = Client()
client.login(username="******", password="******")
resp = client.get(url)
assert resp.status_code == 302
# verify that task ID is saved in the cache (set by the schedule call)
task_id = cache.get("CISCO_EOX_API_SYN_IN_PROGRESS", "")
assert task_id != ""
def test_login_only_mode_configuration(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_login_only_mode() is False
settings.set_login_only_mode(True)
assert settings.is_login_only_mode() is True
settings.set_login_only_mode(False)
assert settings.is_login_only_mode() is False
def test_periodic_sync_enabled(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_periodic_sync_enabled() is False
settings.set_periodic_sync_enabled(True)
assert settings.is_periodic_sync_enabled() is True
settings.set_periodic_sync_enabled(False)
assert settings.is_periodic_sync_enabled() is False
def test_cisco_api_enabled(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_cisco_api_enabled() is False
settings.set_cisco_api_enabled(True)
assert settings.is_cisco_api_enabled() is True
settings.set_cisco_api_enabled(False)
assert settings.is_cisco_api_enabled() is False
def test_config_options_cache():
# check that cache key doesn't exist
assert cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY) is None
# create object
AppSettings()
# key exists and contains a dictionary
assert cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY) is not None
assert type(cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY)) is dict
def login_required_if_login_only_mode(request):
"""
Test if the login only mode is enabled. If this is the case, test if a user is authentication. If this is not the
case, redirect to the login form.
"""
login_only_mode = cache.get("LOGIN_ONLY_MODE_SETTING", None)
if not login_only_mode:
# key not in cache
app_settings = AppSettings()
login_only_mode = app_settings.is_login_only_mode()
# add setting to cache
cache.set("LOGIN_ONLY_MODE_SETTING", login_only_mode, 60 * 60)
if login_only_mode:
if not request.user.is_authenticated:
return True
return False
def login_required_if_login_only_mode(request):
"""
Test if the login only mode is enabled. If this is the case, test if a user is authentication. If this is not the
case, redirect to the login form.
"""
login_only_mode = cache.get("LOGIN_ONLY_MODE_SETTING", None)
if not login_only_mode:
# key not in cache
app_settings = AppSettings()
login_only_mode = app_settings.is_login_only_mode()
# add setting to cache
cache.set("LOGIN_ONLY_MODE_SETTING", login_only_mode, 60 * 60)
if login_only_mode:
if not request.user.is_authenticated():
return True
return False
def test_api_call_error(self, monkeypatch):
# force API failure
def mock_response():
raise CiscoApiCallFailed("The API is broken")
monkeypatch.setattr(api_crawler, "update_cisco_eox_database", lambda query: mock_response())
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("yxcz")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message", None) is None
assert task.info.get("error_message") == "Cisco EoX API call failed (The API is broken)"
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"
def test_api_not_reachable_while_task_synchronization(self, monkeypatch):
monkeypatch.setattr(utils, "check_cisco_eox_api_access", lambda x, y,z: False)
expected_detail_msg = "The synchronization with the Cisco EoX API was not started."
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("yxcz")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message", None) is None
assert task.info.get("error_message") == "Cannot access the Cisco API. Please ensure that the server is " \
"connected to the internet and that the authentication settings are " \
"valid."
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"
assert NotificationMessage.objects.all().first().detailed_message == expected_detail_msg
def test_periodic_task_enabled_state(self, monkeypatch):
self.mock_api_call(monkeypatch)
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(False)
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == "task not enabled"
app.set_periodic_sync_enabled(True)
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") != "task not enabled"
def test_credentials_not_found(self, monkeypatch):
# force API failure
def mock_response():
raise CredentialsNotFoundException("Something is wrong with the credentials handling")
monkeypatch.setattr(api_crawler, "update_cisco_eox_database", lambda query: mock_response())
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("yxcz")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message", None) is None
assert task.info.get("error_message") == "Invalid credentials for Cisco EoX API or insufficient access " \
"rights (Something is wrong with the credentials handling)"
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"
def test_execute_task_to_synchronize_cisco_eox_states_with_failed_api_query(
self, monkeypatch):
def raise_ciscoapicallfailed():
raise CiscoApiCallFailed("Cisco API call failed message")
monkeypatch.setattr(utils, "check_cisco_eox_api_access",
lambda x, y, z: True)
monkeypatch.setattr(
cisco_eox_api_crawler,
"get_raw_api_data",
lambda api_query=None, year=None: raise_ciscoapicallfailed())
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("yxcz")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
expected_status_message = "<p style=\"text-align: left;\">The following queries were executed:<br>" \
"<ul style=\"text-align: left;\"><li class=\"text-danger\">" \
"<code>yxcz</code> (failed, Cisco API call failed message)</li></ul></p>"
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_status_message
assert NotificationMessage.objects.count(
) == 1, "Task should create a Notification Message"
nm = NotificationMessage.objects.first()
assert nm.type == NotificationMessage.MESSAGE_ERROR, "Should be an error message, because all queries failed"
def load_test_cisco_api_credentials():
with open(CISCO_API_TEST_CREDENTIALS_FILE) as f:
cred = json.loads(f.read())
app = AppSettings()
app.set_cisco_api_client_id(cred["client_id"])
app.set_cisco_api_client_secret(cred["client_secret"])
def test_api_check_failed(self, monkeypatch):
# force API failure
class MockSession:
def get(self, *args, **kwargs):
raise Exception("The API is broken")
monkeypatch.setattr(requests, "Session", MockSession)
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("yxcz")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
expected_status_message = "<p style=\"text-align: left;\">The following queries were executed:<br>" \
"<ul style=\"text-align: left;\"><li class=\"text-danger\"><code>yxcz</code> " \
"(failed, cannot contact API endpoint at " \
"https://api.cisco.com/supporttools/eox/rest/5/EOXByProductID/1/yxcz)</li></ul></p>"
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_status_message
assert NotificationMessage.objects.count(
) == 1, "Task should create a Notification Message"
nm = NotificationMessage.objects.first()
assert nm.type == NotificationMessage.MESSAGE_ERROR, "Should be an error message, because all queries failed"
def login_user(request):
"""login user
:param request:
:return:
"""
app_config = AppSettings()
context = {
"login_only_mode": app_config.is_login_only_mode()
}
if request.user.is_authenticated():
return HttpResponseRedirect(reverse("productdb:home"))
if request.GET:
context["next"] = request.GET['next']
else:
context["next"] = None
if request.method == 'POST':
# authenticate user
username = request.POST['username']
password = request.POST['password']
user = authenticate(username=username, password=password)
if user is not None:
if user.is_active:
login(request, user)
if context["next"] and not context["next"].startswith("/productdb/login"):
return HttpResponseRedirect(context["next"])
else:
return HttpResponseRedirect(reverse("productdb:home"))
else:
context["message"] = "User account was disabled.<br>Please contact the administrator."
else:
context["message"] = "Login failed, invalid credentials"
return render(request, "django_project/login.html", context=context)
def login_user(request):
"""login user
:param request:
:return:
"""
app_config = AppSettings()
context = {"login_only_mode": app_config.is_login_only_mode()}
if request.user.is_authenticated():
return HttpResponseRedirect(reverse("productdb:home"))
if request.GET:
context["next"] = request.GET['next']
else:
context["next"] = None
if request.method == 'POST':
# authenticate user
username = request.POST['username']
password = request.POST['password']
user = authenticate(username=username, password=password)
if user is not None:
if user.is_active:
login(request, user)
if context["next"] and not context["next"].startswith(
"/productdb/login"):
return HttpResponseRedirect(context["next"])
else:
return HttpResponseRedirect(reverse("productdb:home"))
else:
context[
"message"] = "User account was disabled.<br>Please contact the administrator."
else:
context["message"] = "Login failed, invalid credentials"
return render(request, "django_project/login.html", context=context)
def test_api_check_failed(self, monkeypatch):
# force API failure
def mock_response():
raise Exception("The API is broken")
monkeypatch.setattr(requests, "get", lambda x, headers: mock_response())
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("yxcz")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message", None) is None
assert task.info.get("error_message") == "Cannot access the Cisco API. Please ensure that the server is " \
"connected to the internet and that the authentication settings are " \
"valid."
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"
def test_offline_valid_update_cisco_eox_database_with_create_flag_and_invalid_blacklist(self, monkeypatch):
# mock the underlying GET request
def mock_response():
r = Response()
r.status_code = 200
with open("app/ciscoeox/tests/data/cisco_eox_response_page_1_of_1.json") as f:
r._content = f.read().encode("utf-8")
return r
monkeypatch.setattr(requests, "get", lambda x, headers: mock_response())
app = AppSettings()
app.set_product_blacklist_regex("*-WS$")
result = api_crawler.update_cisco_eox_database("WS-C2950G-48-EI-WS")
assert len(result) == 3, "Three products should be imported"
# every product should contain the following values
for e in result:
assert "blacklist" in e
assert "updated" in e
assert "created" in e
assert "PID" in e
assert "message" in e
if e["PID"] == "WS-C2950G-48-EI-WS":
# Nothing is blacklisted, input of an invalid regular expression should not be possible
assert e["blacklist"] is False
assert e["updated"] is True
assert e["created"] is True
assert type(e["PID"]) == str
assert e["message"] is None
else:
assert e["blacklist"] is False
assert e["updated"] is True
assert e["created"] is True
assert type(e["PID"]) == str
assert e["message"] is None
assert Product.objects.count() == 3, "No products are created, because the creation mode is disabled by default"
def status(request):
"""
Status page for the Product Database
"""
app_config = AppSettings()
is_cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {"is_cisco_api_enabled": is_cisco_api_enabled}
if is_cisco_api_enabled:
# test access (once every 30 minutes)
cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)
# defaults, overwritten if an exception is thrown
cisco_eox_api_available = True
cisco_eox_api_message = "successful connected to the Cisco EoX API"
if not cisco_eox_api_test_successful:
try:
result = utils.check_cisco_eox_api_access(
client_id=app_config.get_cisco_api_client_id(),
client_secret=app_config.get_cisco_api_client_secret(),
drop_credentials=False)
cache.set("CISCO_EOX_API_TEST", result, 60 * 30)
except Exception as ex:
cisco_eox_api_available = True
cisco_eox_api_message = str(ex)
context["cisco_eox_api_available"] = cisco_eox_api_available
context["cisco_eox_api_message"] = cisco_eox_api_message
# determine worker status
context['worker_status'] = mark_safe(utils.get_celery_worker_state_html())
return render(request, "config/status.html", context=context)
def test_auto_create_new_products(self):
settings = AppSettings()
# get values
value = settings.is_auto_create_new_products()
assert value is False
# set values
settings.set_auto_create_new_products(True)
value = settings.is_auto_create_new_products()
assert value is True
def test_cisco_api_client_id(self):
settings = AppSettings()
# get values
value = settings.get_cisco_api_client_id()
assert value == "PlsChgMe"
# set values
test_client_id = "test_id"
settings.set_cisco_api_client_id(test_client_id)
value = settings.get_cisco_api_client_id()
assert value == test_client_id
def test_product_blacklist_regex_from_config(self):
settings = AppSettings()
# get values
value = settings.get_product_blacklist_regex()
assert value == ""
# set values
blacklist_entries = "test\nanother\ntest"
settings.set_product_blacklist_regex(blacklist_entries)
value = settings.get_product_blacklist_regex()
assert value == blacklist_entries
def test_cisco_api_client_secret(self):
settings = AppSettings()
# get values
value = settings.get_cisco_api_client_secret()
assert value == "PlsChgMe"
# set values
test_client_secret = "test_secret"
settings.set_cisco_api_client_secret(test_client_secret)
value = settings.get_cisco_api_client_secret()
assert value == test_client_secret
def test_internal_product_id_label(self):
settings = AppSettings()
# get values
value = settings.get_internal_product_id_label()
assert value == "Internal Product ID"
# set values
test_internal_product_label = "My custom label"
settings.set_internal_product_id_label(test_internal_product_label)
value = settings.get_internal_product_id_label()
assert value == test_internal_product_label
def test_cisco_eox_wait_time(self):
settings = AppSettings()
# get value
value = settings.get_cisco_eox_api_sync_wait_time()
assert value == "5"
# set values
test_cisco_eox_wait_time = "10"
settings.set_cisco_eox_api_sync_wait_time(test_cisco_eox_wait_time)
value = settings.get_cisco_eox_api_sync_wait_time()
assert value == test_cisco_eox_wait_time
def test_cisco_eox_api_auto_sync_last_execution_result(self):
settings = AppSettings()
# get values
value = settings.get_cisco_eox_api_auto_sync_last_execution_result()
assert value is None
# set values
test_last_execution_result = "everything is fine"
settings.set_cisco_eox_api_auto_sync_last_execution_result(
test_last_execution_result)
value = settings.get_cisco_eox_api_auto_sync_last_execution_result()
assert value == test_last_execution_result
def test_manual_task(self, monkeypatch):
self.mock_api_call(monkeypatch)
# try to execute it, while no auto-sync is enabled
app = AppSettings()
app.set_periodic_sync_enabled(False)
app.set_cisco_eox_api_queries("WS-C2960-*")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True)
expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>The following products are ' \
'affected by this update:</p><ul><li>create the Product <code>WS-C2950G-48-EI-WS</code> ' \
'in the database</li><li>create the Product <code>WS-C2950T-48-SI-WS</code> in the ' \
'database</li><li>create the Product <code>WS-C2950G-24-EI</code> in the database</li>' \
'</ul></div>'
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_result
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"
assert Product.objects.count() == 3, "Three products are part of the update"
# test no changes required
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True)
expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>No changes required.</div>'
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_result
assert NotificationMessage.objects.count() == 2, "Task should create a Notification Message"
assert Product.objects.count() == 3, "Three products are part of the update"
# test update required
p = Product.objects.get(product_id="WS-C2950G-24-EI")
p.eox_update_time_stamp = datetime.date(1999, 1, 1)
p.save()
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True)
expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>The following products are ' \
'affected by this update:</p><ul><li>update the Product data for <code>WS-C2950G-24-EI' \
'</code></li></ul></div>'
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_result
assert NotificationMessage.objects.count() == 3, "Task should create a Notification Message"
assert Product.objects.count() == 3, "Three products are part of the update"
def test_manual_task_with_multiple_blacklist_entries(self, monkeypatch):
self.mock_api_call(monkeypatch)
# try to execute it, while no auto-sync is enabled
app = AppSettings()
app.set_periodic_sync_enabled(False)
app.set_cisco_eox_api_queries("WS-C2960-*")
app.set_product_blacklist_regex("WS-C2950G-48-EI-WS;WS-C2950G-24-EI")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True)
expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>The following products are ' \
'affected by this update:</p><ul><li>Product data for <code>WS-C2950G-48-EI-WS</code> ' \
'ignored</li><li>create the Product <code>WS-C2950T-48-SI-WS</code> in the ' \
'database</li><li>Product data for <code>WS-C2950G-24-EI</code> ignored</li>' \
'</ul></div>'
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_result
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"
assert Product.objects.count() == 1, "Only a single product is imported"
def update_cisco_eox_database(api_query):
"""
synchronizes the local database with the Cisco EoX API using the specified query
:param api_query: single query that is send to the Cisco EoX API
:raises CiscoApiCallFailed: exception raised if Cisco EoX API call failed
:return: list of dictionary that describe the updates to the database
"""
if type(api_query) is not str:
raise ValueError("api_query must be a string value")
# load application settings and check, that the API is enabled
app_settings = AppSettings()
if not app_settings.is_cisco_api_enabled():
msg = "Cisco API access not enabled"
logger.warn(msg)
raise CiscoApiCallFailed(msg)
blacklist_raw_string = app_settings.get_product_blacklist_regex()
create_missing = app_settings.is_auto_create_new_products()
# clean blacklist string and remove empty statements
# (split lines, if any and split the string by semicolon)
blacklist = []
for e in [e.split(";") for e in blacklist_raw_string.splitlines()]:
blacklist += e
blacklist = [e for e in blacklist if e != ""]
# start Cisco EoX API query
logger.info("Query EoX database: %s" % api_query)
eoxapi = CiscoEoxApi()
eoxapi.load_client_credentials()
results = []
try:
max_pages = 999
current_page = 1
result_pages = 0
while current_page <= max_pages:
logger.info("Executing API query '%s' on page '%d" % (api_query, current_page))
# will raise a CiscoApiCallFailed exception on error
eoxapi.query_product(product_id=api_query, page=current_page)
if current_page == 1:
result_pages = eoxapi.amount_of_pages()
logger.info("Initial query returns %d page(s)" % result_pages)
records = eoxapi.get_eox_records()
# check that the query has valid results
if eoxapi.get_page_record_count() > 0:
# processing records
for record in records:
result_record = {}
pid = record["EOLProductID"]
result_record["PID"] = pid
result_record["created"] = False
result_record["updated"] = False
result_record["message"] = None
logger.info("processing product '%s'..." % pid)
pid_in_database = product_id_in_database(pid)
# check if the product ID is blacklisted by a regular expression
pid_blacklisted = False
for regex in blacklist:
try:
if re.search(regex, pid, re.I):
pid_blacklisted = True
break
except:
# silently ignore the issue, invalid regular expressions are handled by the settings form
logger.info("invalid regular expression: %s" % regex)
# ignore if the product id is not in the database
if pid_blacklisted and not pid_in_database:
logger.info("Product '%s' blacklisted... no further processing" % pid)
result_record.update({"blacklist": True})
else:
res = {}
try:
res = update_local_db_based_on_record(record, create_missing)
except ValidationError:
logger.error(
"invalid data received from Cisco API, cannot save data object for '%s'" % pid,
exc_info=True,
)
result_record["message"] = "invalid data received from Cisco EoX API, import incomplete"
finally:
res["blacklist"] = False
result_record.update(res)
results.append(result_record)
if current_page == result_pages:
break
else:
current_page += 1
# filter empty result strings
if len(results) == 0:
results = [
{
"PID": None,
"blacklist": False,
"updated": False,
"created": False,
"message": "No product update required",
}
]
except ConnectionFailedException:
logger.error("Query failed, server not reachable: %s" % api_query, exc_info=True)
raise
except CiscoApiCallFailed:
logger.fatal("Query failed: %s" % api_query, exc_info=True)
raise
return results
