def test_manual_task_with_multiple_blacklist_entries(self, monkeypatch):
self.mock_api_call(monkeypatch)
# try to execute it, while no auto-sync is enabled
app = AppSettings()
app.set_periodic_sync_enabled(False)
app.set_cisco_eox_api_queries("WS-C2960-*")
app.set_product_blacklist_regex("WS-C2950G-48-EI-WS;WS-C2950G-24-EI")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(
ignore_periodic_sync_flag=True)
expected_result = [
'<p style="text-align: left;">The following queries were executed:<br>'
'<ul style="text-align: left;"><li><code>WS-C2960-*</code> (<b>affects 3 products</b>, '
'success)</li></ul>'
]
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
for er in expected_result:
assert er in task.info.get("status_message")
assert NotificationMessage.objects.count(
) == 1, "Task should create a Notification Message"
assert Product.objects.count(
) == 1, "Only a single product is imported"
nm = NotificationMessage.objects.first()
assert nm.type == NotificationMessage.MESSAGE_SUCCESS, "Incomplete configuration, should throw a warning "
def test_cisco_eox_queries(self):
settings = AppSettings()
# get values
value = settings.get_cisco_eox_api_queries()
assert value == ""
# set values
queries = "test"
settings.set_cisco_eox_api_queries(queries)
value = settings.get_cisco_eox_api_queries()
assert value == queries
assert settings.get_cisco_eox_api_queries_as_list() == ["test"]
queries = "test\nother_test"
settings.set_cisco_eox_api_queries(queries)
value = settings.get_cisco_eox_api_queries()
assert value == queries
assert settings.get_cisco_eox_api_queries_as_list() == [
"test", "other_test"
]
queries = "test;xyz\nother_test"
settings.set_cisco_eox_api_queries(queries)
value = settings.get_cisco_eox_api_queries()
assert value == queries
assert settings.get_cisco_eox_api_queries_as_list() == [
"test", "xyz", "other_test"
]
def perform_product_check(self):
"""perform the product check and populate the ProductCheckEntries"""
unique_products = [line.strip() for line in set(self.input_product_ids_list) if line.strip() != ""]
amounts = Counter(self.input_product_ids_list)
# clean all entries
self.productcheckentry_set.all().delete()
for input_product_id in unique_products:
product_entry, _ = ProductCheckEntry.objects.get_or_create(
input_product_id=input_product_id,
product_check=self
)
product_entry.amount = amounts[input_product_id]
product_entry.discover_product_list_values()
product_entry.save()
# increments statistics
settings = AppSettings()
settings.set_amount_of_product_checks(settings.get_amount_of_product_checks() + 1)
settings.set_amount_of_unique_product_check_entries(settings.get_amount_of_unique_product_check_entries() +
len(unique_products))
self.save()
def test_execute_task_to_synchronize_cisco_eox_states_with_failed_api_query(
self, monkeypatch):
def raise_ciscoapicallfailed():
raise CiscoApiCallFailed("Cisco API call failed message")
monkeypatch.setattr(utils, "check_cisco_eox_api_access",
lambda x, y, z: True)
monkeypatch.setattr(
cisco_eox_api_crawler,
"get_raw_api_data",
lambda api_query=None, year=None: raise_ciscoapicallfailed())
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("yxcz")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
expected_status_message = "<p style=\"text-align: left;\">The following queries were executed:<br>" \
"<ul style=\"text-align: left;\"><li class=\"text-danger\">" \
"<code>yxcz</code> (failed, Cisco API call failed message)</li></ul></p>"
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_status_message
assert NotificationMessage.objects.count(
) == 1, "Task should create a Notification Message"
nm = NotificationMessage.objects.first()
assert nm.type == NotificationMessage.MESSAGE_ERROR, "Should be an error message, because all queries failed"
def test_api_check_failed(self, monkeypatch):
# force API failure
class MockSession:
def get(self, *args, **kwargs):
raise Exception("The API is broken")
monkeypatch.setattr(requests, "Session", MockSession)
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("yxcz")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
expected_status_message = "<p style=\"text-align: left;\">The following queries were executed:<br>" \
"<ul style=\"text-align: left;\"><li class=\"text-danger\"><code>yxcz</code> " \
"(failed, cannot contact API endpoint at " \
"https://api.cisco.com/supporttools/eox/rest/5/EOXByProductID/1/yxcz)</li></ul></p>"
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_status_message
assert NotificationMessage.objects.count(
) == 1, "Task should create a Notification Message"
nm = NotificationMessage.objects.first()
assert nm.type == NotificationMessage.MESSAGE_ERROR, "Should be an error message, because all queries failed"
def load_client_credentials(self):
logger.debug("load client credentials from configuration")
app_settings = AppSettings()
# load client credentials
self.client_id = app_settings.get_cisco_api_client_id()
self.client_secret = app_settings.get_cisco_api_client_secret()
def load_test_cisco_api_credentials():
with open(CISCO_API_TEST_CREDENTIALS_FILE) as f:
cred = json.loads(f.read())
app = AppSettings()
app.set_cisco_api_client_id(cred["client_id"])
app.set_cisco_api_client_secret(cred["client_secret"])
def handle(self, *args, **kwargs):
app_settings = AppSettings()
if not app_settings.is_cisco_api_enabled():
raise CommandError(
"Please configure the Cisco EoX API in the settings prior running this task"
)
# check if task is already running, if so print status and continue
task_id = cache.get("CISCO_EOX_INITIAL_SYN_IN_PROGRESS", None)
if task_id is None:
# start initial import
eta = now() + timedelta(seconds=3)
task = tasks.initial_sync_with_cisco_eox_api.apply_async(
eta=eta, args=(kwargs["years"], ))
cache.set("CISCO_EOX_INITIAL_SYN_IN_PROGRESS", task.id,
60 * 60 * 48)
cache.set("CISCO_EOX_INITIAL_SYN_LAST_RUN", task.id, 60 * 60 * 48)
self.stdout.write("Task successful started...")
else:
msg = get_task_state_message(task_id)
raise CommandError("initial import already running... \n%s" % msg)
def get_internal_product_id_label(request):
app_config = AppSettings()
return {
"INTERNAL_PRODUCT_ID_LABEL":
app_config.get_internal_product_id_label(),
"STAT_AMOUNT_OF_PRODUCT_CHECKS":
app_config.get_amount_of_product_checks(),
"STAT_AMOUNT_OF_UNIQUE_PRODUCT_CHECK_ENTRIES":
app_config.get_amount_of_unique_product_check_entries()
}
def test_config_options_cache():
# check that cache key doesn't exist
assert cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY) is None
# create object
AppSettings()
# key exists and contains a dictionary
assert cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY) is not None
assert type(cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY)) is dict
def get_raw_api_data(api_query):
"""
returns all EoX records for a specific query (from all pages)
:param api_query: single query that is send to the Cisco EoX API
:raises CiscoApiCallFailed: exception raised if Cisco EoX API call failed
:return: list that contains all EoX records from the Cisco EoX API
"""
if type(api_query) is not str:
raise ValueError("api_query must be a string value")
# load application settings and check, that the API is enabled
app_settings = AppSettings()
if not app_settings.is_cisco_api_enabled():
msg = "Cisco API access not enabled"
logger.warning(msg)
raise CiscoApiCallFailed(msg)
# start Cisco EoX API query
logger.info("send query to Cisco EoX database: %s" % api_query)
eoxapi = CiscoEoxApi()
eoxapi.load_client_credentials()
results = []
try:
current_page = 1
result_pages = 999
while current_page <= result_pages:
if current_page == 1:
logger.info("Executing API query '%s' on first page" % api_query)
else:
logger.info("Executing API query '%s' on page '%d" % (api_query, current_page))
# will raise a CiscoApiCallFailed exception on error
eoxapi.query_product(product_id=api_query, page=current_page)
result_pages = eoxapi.amount_of_pages()
if eoxapi.get_page_record_count() > 0:
results.extend(eoxapi.get_eox_records())
current_page += 1
except ConnectionFailedException:
logger.error("Query failed, server not reachable: %s" % api_query, exc_info=True)
raise
except CiscoApiCallFailed:
logger.fatal("Query failed: %s" % api_query, exc_info=True)
raise
return results
def test_periodic_sync_enabled(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_periodic_sync_enabled() is False
settings.set_periodic_sync_enabled(True)
assert settings.is_periodic_sync_enabled() is True
settings.set_periodic_sync_enabled(False)
assert settings.is_periodic_sync_enabled() is False
def test_cisco_api_enabled(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_cisco_api_enabled() is False
settings.set_cisco_api_enabled(True)
assert settings.is_cisco_api_enabled() is True
settings.set_cisco_api_enabled(False)
assert settings.is_cisco_api_enabled() is False
def test_write_datetime_to_settings_file(self):
settings = AppSettings()
# get a configuration value (default in the global section)
now = datetime.now()
value = now.isoformat()
settings.set_cisco_eox_api_auto_sync_last_execution_time(value)
read_value = settings.get_cisco_eox_api_auto_sync_last_execution_time()
assert value == read_value
assert now == parse_datetime(read_value)
def use_test_api_configuration():
app = AppSettings()
with open(CREDENTIALS_FILE) as f:
content = json.loads(f.read())
app.set_cisco_api_enabled(True)
app.set_product_blacklist_regex("")
app.set_cisco_eox_api_queries("")
app.set_auto_create_new_products(True)
app.set_periodic_sync_enabled(False)
app.set_cisco_api_client_id(content.get("client_id", "dummy_id"))
app.set_cisco_api_client_id(content.get("client_secret", "dummy_secret"))
def use_test_api_configuration():
app = AppSettings()
app.set_cisco_api_enabled(True)
app.set_product_blacklist_regex("")
app.set_cisco_eox_api_queries("")
app.set_auto_create_new_products(True)
app.set_periodic_sync_enabled(False)
app.set_cisco_api_client_id(
os.getenv("TEST_CISCO_API_CLIENT_ID", "dummy_id"))
app.set_cisco_api_client_secret(
os.getenv("TEST_CISCO_API_CLIENT_SECRET", "dummy_secret"))
def test_login_only_mode_configuration(self):
# create new AppSettings object and create defaults
settings = AppSettings()
assert settings.is_login_only_mode() is False
settings.set_login_only_mode(True)
assert settings.is_login_only_mode() is True
settings.set_login_only_mode(False)
assert settings.is_login_only_mode() is False
def test_auto_create_new_products(self):
settings = AppSettings()
# get values
value = settings.is_auto_create_new_products()
assert value is False
# set values
settings.set_auto_create_new_products(True)
value = settings.is_auto_create_new_products()
assert value is True
def mock_cisco_eox_api_access_broken(monkeypatch):
app = AppSettings()
app.set_cisco_api_enabled(True)
app.set_cisco_api_client_id("client_id")
app.set_cisco_api_client_id("client_secret")
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("")
app.set_product_blacklist_regex("")
app.set_auto_create_new_products(True)
monkeypatch.setattr(utils, "check_cisco_eox_api_access",
lambda client_id, client_secret, drop_credentials=False: False)
def cisco_eox_populate_product_lc_state_sync_field():
"""
Periodic job to populate the lc_state_sync field in the Products, which shows that the product lifecycle data are
automatically synchronized against the Cisco EoX API in this case
:return:
"""
try:
cis_vendor = Vendor.objects.get(name__istartswith="Cisco")
except:
# Vendor doesn't exist, no steps required
logger.fatal(
"Vendor \"Cisco Systems\" not found in database, please check your installation"
)
return {"error": "Vendor \"Cisco Systems\" not found in database"}
cisco_products = Product.objects.filter(vendor=cis_vendor)
if cisco_products.count() != 0:
app_config = AppSettings()
queries = app_config.get_cisco_eox_api_queries_as_list()
# escape the query strings
queries = [re.escape(e) for e in queries]
# convert the wildcard values
queries = [e.replace("\\*", ".*") for e in queries]
queries = ["^" + e + "$" for e in queries]
with transaction.atomic():
# reset all entries for the vendor
pl = Product.objects.filter(vendor=cis_vendor)
for p in pl:
p.lc_state_sync = False
p.save()
# only set the state sync to true if the periodic synchronization is enabled
if app_config.is_periodic_sync_enabled():
for query in queries:
pl = Product.objects.filter(product_id__regex=query,
vendor=cis_vendor)
for p in pl:
p.lc_state_sync = True
p.save()
return {"status": "Database updated"}
else:
return {
"error":
"No Products associated to \"Cisco Systems\" found in database"
}
def status(request):
"""
Status page for the Product Database
"""
app_config = AppSettings()
is_cisco_api_enabled = app_config.is_cisco_api_enabled()
context = {
"is_cisco_api_enabled": is_cisco_api_enabled
}
if is_cisco_api_enabled:
# test access (once every 30 minutes)
cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)
# defaults, overwritten if an exception is thrown
cisco_eox_api_available = True
cisco_eox_api_message = "successful connected to the Cisco EoX API"
if not cisco_eox_api_test_successful:
try:
result = utils.check_cisco_eox_api_access(client_id=app_config.get_cisco_api_client_id(),
client_secret=app_config.get_cisco_api_client_secret(),
drop_credentials=False)
cache.set("CISCO_EOX_API_TEST", result, 60 * 30)
except Exception as ex:
cisco_eox_api_available = True
cisco_eox_api_message = str(ex)
context["cisco_eox_api_available"] = cisco_eox_api_available
context["cisco_eox_api_message"] = cisco_eox_api_message
# determine worker status
state = celery.is_worker_active()
if state and not settings.DEBUG:
worker_status = """
<div class="alert alert-success" role="alert">
<span class="fa fa-info-circle"></span>
Backend worker found.
</div>"""
else:
worker_status = """
<div class="alert alert-danger" role="alert">
<span class="fa fa-exclamation-circle"></span>
No backend worker found, asynchronous and scheduled tasks are not executed.
</div>"""
context['worker_status'] = mark_safe(worker_status)
return render(request, "config/status.html", context=context)
def test_product_blacklist_regex_from_config(self):
settings = AppSettings()
# get values
value = settings.get_product_blacklist_regex()
assert value == ""
# set values
blacklist_entries = "test\nanother\ntest"
settings.set_product_blacklist_regex(blacklist_entries)
value = settings.get_product_blacklist_regex()
assert value == blacklist_entries
def test_cisco_api_client_secret(self):
settings = AppSettings()
# get values
value = settings.get_cisco_api_client_secret()
assert value == "PlsChgMe"
# set values
test_client_secret = "test_secret"
settings.set_cisco_api_client_secret(test_client_secret)
value = settings.get_cisco_api_client_secret()
assert value == test_client_secret
def test_internal_product_id_label(self):
settings = AppSettings()
# get values
value = settings.get_internal_product_id_label()
assert value == "Internal Product ID"
# set values
test_internal_product_label = "My custom label"
settings.set_internal_product_id_label(test_internal_product_label)
value = settings.get_internal_product_id_label()
assert value == test_internal_product_label
def test_cisco_eox_wait_time(self):
settings = AppSettings()
# get value
value = settings.get_cisco_eox_api_sync_wait_time()
assert value == "5"
# set values
test_cisco_eox_wait_time = "10"
settings.set_cisco_eox_api_sync_wait_time(test_cisco_eox_wait_time)
value = settings.get_cisco_eox_api_sync_wait_time()
assert value == test_cisco_eox_wait_time
def test_cisco_api_client_id(self):
settings = AppSettings()
# get values
value = settings.get_cisco_api_client_id()
assert value == "PlsChgMe"
# set values
test_client_id = "test_id"
settings.set_cisco_api_client_id(test_client_id)
value = settings.get_cisco_api_client_id()
assert value == test_client_id
def test_cisco_eox_api_auto_sync_last_execution_result(self):
settings = AppSettings()
# get values
value = settings.get_cisco_eox_api_auto_sync_last_execution_result()
assert value is None
# set values
test_last_execution_result = "everything is fine"
settings.set_cisco_eox_api_auto_sync_last_execution_result(
test_last_execution_result)
value = settings.get_cisco_eox_api_auto_sync_last_execution_result()
assert value == test_last_execution_result
def update_cisco_eox_records(records):
"""
update given database records from the Cisco EoX v5 API
:param records:
:return:
"""
app_config = AppSettings()
blacklist_raw_string = app_config.get_product_blacklist_regex()
create_missing = app_config.is_auto_create_new_products()
# build blacklist from configuration
blacklist = []
for e in [e.split(";") for e in blacklist_raw_string.splitlines()]:
blacklist += e
blacklist = [e for e in blacklist if e != ""]
counter = 0
messages = {}
for record in records:
blacklisted = False
for regex in blacklist:
try:
if re.search(regex, record["EOLProductID"], re.I):
blacklisted = True
break
except:
logger.warning("invalid regular expression in blacklist: %s" % regex)
if not blacklisted:
try:
message = cisco_eox_api_crawler.update_local_db_based_on_record(record, create_missing)
if message:
messages[record["EOLProductID"]] = message
except ValidationError as ex:
logger.error("invalid data received from Cisco API, cannot save data object for "
"'%s' (%s)" % (record, str(ex)), exc_info=True)
else:
messages[record["EOLProductID"]] = " Product record ignored"
counter += 1
return {
"count": counter,
"messages": messages
}
def test_manual_task(self, monkeypatch):
self.mock_api_call(monkeypatch)
# try to execute it, while no auto-sync is enabled
app = AppSettings()
app.set_periodic_sync_enabled(False)
app.set_cisco_eox_api_queries("WS-C2960-*")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True)
expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>The following products are ' \
'affected by this update:</p><ul><li>create the Product <code>WS-C2950G-48-EI-WS</code> ' \
'in the database</li><li>create the Product <code>WS-C2950T-48-SI-WS</code> in the ' \
'database</li><li>create the Product <code>WS-C2950G-24-EI</code> in the database</li>' \
'</ul></div>'
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_result
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"
assert Product.objects.count() == 3, "Three products are part of the update"
# test no changes required
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True)
expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>No changes required.</div>'
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_result
assert NotificationMessage.objects.count() == 2, "Task should create a Notification Message"
assert Product.objects.count() == 3, "Three products are part of the update"
# test update required
p = Product.objects.get(product_id="WS-C2950G-24-EI")
p.eox_update_time_stamp = datetime.date(1999, 1, 1)
p.save()
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True)
expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>The following products are ' \
'affected by this update:</p><ul><li>update the Product data for <code>WS-C2950G-24-EI' \
'</code></li></ul></div>'
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == expected_result
assert NotificationMessage.objects.count() == 3, "Task should create a Notification Message"
assert Product.objects.count() == 3, "Three products are part of the update"
def test_periodic_task_without_queries(self, monkeypatch):
self.mock_api_call(monkeypatch)
# test automatic trigger
app = AppSettings()
app.set_periodic_sync_enabled(True)
app.set_cisco_eox_api_queries("")
task = tasks.execute_task_to_synchronize_cisco_eox_states.delay()
assert task is not None
assert task.status == "SUCCESS", task.traceback
assert task.state == TaskState.SUCCESS
assert task.info.get("status_message") == "No Cisco EoX API queries configured."
assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"