def test_manual_task_with_multiple_blacklist_entries(self, monkeypatch): self.mock_api_call(monkeypatch) # try to execute it, while no auto-sync is enabled app = AppSettings() app.set_periodic_sync_enabled(False) app.set_cisco_eox_api_queries("WS-C2960-*") app.set_product_blacklist_regex("WS-C2950G-48-EI-WS;WS-C2950G-24-EI") task = tasks.execute_task_to_synchronize_cisco_eox_states.delay( ignore_periodic_sync_flag=True) expected_result = [ '<p style="text-align: left;">The following queries were executed:<br>' '<ul style="text-align: left;"><li><code>WS-C2960-*</code> (<b>affects 3 products</b>, ' 'success)</li></ul>' ] assert task is not None assert task.status == "SUCCESS", task.traceback assert task.state == TaskState.SUCCESS for er in expected_result: assert er in task.info.get("status_message") assert NotificationMessage.objects.count( ) == 1, "Task should create a Notification Message" assert Product.objects.count( ) == 1, "Only a single product is imported" nm = NotificationMessage.objects.first() assert nm.type == NotificationMessage.MESSAGE_SUCCESS, "Incomplete configuration, should throw a warning "
def test_cisco_eox_queries(self): settings = AppSettings() # get values value = settings.get_cisco_eox_api_queries() assert value == "" # set values queries = "test" settings.set_cisco_eox_api_queries(queries) value = settings.get_cisco_eox_api_queries() assert value == queries assert settings.get_cisco_eox_api_queries_as_list() == ["test"] queries = "test\nother_test" settings.set_cisco_eox_api_queries(queries) value = settings.get_cisco_eox_api_queries() assert value == queries assert settings.get_cisco_eox_api_queries_as_list() == [ "test", "other_test" ] queries = "test;xyz\nother_test" settings.set_cisco_eox_api_queries(queries) value = settings.get_cisco_eox_api_queries() assert value == queries assert settings.get_cisco_eox_api_queries_as_list() == [ "test", "xyz", "other_test" ]
def perform_product_check(self): """perform the product check and populate the ProductCheckEntries""" unique_products = [line.strip() for line in set(self.input_product_ids_list) if line.strip() != ""] amounts = Counter(self.input_product_ids_list) # clean all entries self.productcheckentry_set.all().delete() for input_product_id in unique_products: product_entry, _ = ProductCheckEntry.objects.get_or_create( input_product_id=input_product_id, product_check=self ) product_entry.amount = amounts[input_product_id] product_entry.discover_product_list_values() product_entry.save() # increments statistics settings = AppSettings() settings.set_amount_of_product_checks(settings.get_amount_of_product_checks() + 1) settings.set_amount_of_unique_product_check_entries(settings.get_amount_of_unique_product_check_entries() + len(unique_products)) self.save()
def test_execute_task_to_synchronize_cisco_eox_states_with_failed_api_query( self, monkeypatch): def raise_ciscoapicallfailed(): raise CiscoApiCallFailed("Cisco API call failed message") monkeypatch.setattr(utils, "check_cisco_eox_api_access", lambda x, y, z: True) monkeypatch.setattr( cisco_eox_api_crawler, "get_raw_api_data", lambda api_query=None, year=None: raise_ciscoapicallfailed()) # test automatic trigger app = AppSettings() app.set_periodic_sync_enabled(True) app.set_cisco_eox_api_queries("yxcz") task = tasks.execute_task_to_synchronize_cisco_eox_states.delay() expected_status_message = "<p style=\"text-align: left;\">The following queries were executed:<br>" \ "<ul style=\"text-align: left;\"><li class=\"text-danger\">" \ "<code>yxcz</code> (failed, Cisco API call failed message)</li></ul></p>" assert task is not None assert task.status == "SUCCESS", task.traceback assert task.state == TaskState.SUCCESS assert task.info.get("status_message") == expected_status_message assert NotificationMessage.objects.count( ) == 1, "Task should create a Notification Message" nm = NotificationMessage.objects.first() assert nm.type == NotificationMessage.MESSAGE_ERROR, "Should be an error message, because all queries failed"
def test_api_check_failed(self, monkeypatch): # force API failure class MockSession: def get(self, *args, **kwargs): raise Exception("The API is broken") monkeypatch.setattr(requests, "Session", MockSession) # test automatic trigger app = AppSettings() app.set_periodic_sync_enabled(True) app.set_cisco_eox_api_queries("yxcz") task = tasks.execute_task_to_synchronize_cisco_eox_states.delay() expected_status_message = "<p style=\"text-align: left;\">The following queries were executed:<br>" \ "<ul style=\"text-align: left;\"><li class=\"text-danger\"><code>yxcz</code> " \ "(failed, cannot contact API endpoint at " \ "https://api.cisco.com/supporttools/eox/rest/5/EOXByProductID/1/yxcz)</li></ul></p>" assert task is not None assert task.status == "SUCCESS", task.traceback assert task.state == TaskState.SUCCESS assert task.info.get("status_message") == expected_status_message assert NotificationMessage.objects.count( ) == 1, "Task should create a Notification Message" nm = NotificationMessage.objects.first() assert nm.type == NotificationMessage.MESSAGE_ERROR, "Should be an error message, because all queries failed"
def load_client_credentials(self): logger.debug("load client credentials from configuration") app_settings = AppSettings() # load client credentials self.client_id = app_settings.get_cisco_api_client_id() self.client_secret = app_settings.get_cisco_api_client_secret()
def load_test_cisco_api_credentials(): with open(CISCO_API_TEST_CREDENTIALS_FILE) as f: cred = json.loads(f.read()) app = AppSettings() app.set_cisco_api_client_id(cred["client_id"]) app.set_cisco_api_client_secret(cred["client_secret"])
def handle(self, *args, **kwargs): app_settings = AppSettings() if not app_settings.is_cisco_api_enabled(): raise CommandError( "Please configure the Cisco EoX API in the settings prior running this task" ) # check if task is already running, if so print status and continue task_id = cache.get("CISCO_EOX_INITIAL_SYN_IN_PROGRESS", None) if task_id is None: # start initial import eta = now() + timedelta(seconds=3) task = tasks.initial_sync_with_cisco_eox_api.apply_async( eta=eta, args=(kwargs["years"], )) cache.set("CISCO_EOX_INITIAL_SYN_IN_PROGRESS", task.id, 60 * 60 * 48) cache.set("CISCO_EOX_INITIAL_SYN_LAST_RUN", task.id, 60 * 60 * 48) self.stdout.write("Task successful started...") else: msg = get_task_state_message(task_id) raise CommandError("initial import already running... \n%s" % msg)
def get_internal_product_id_label(request): app_config = AppSettings() return { "INTERNAL_PRODUCT_ID_LABEL": app_config.get_internal_product_id_label(), "STAT_AMOUNT_OF_PRODUCT_CHECKS": app_config.get_amount_of_product_checks(), "STAT_AMOUNT_OF_UNIQUE_PRODUCT_CHECK_ENTRIES": app_config.get_amount_of_unique_product_check_entries() }
def test_config_options_cache(): # check that cache key doesn't exist assert cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY) is None # create object AppSettings() # key exists and contains a dictionary assert cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY) is not None assert type(cache.get(AppSettings.CONFIG_OPTIONS_DICT_CACHE_KEY)) is dict
def get_raw_api_data(api_query): """ returns all EoX records for a specific query (from all pages) :param api_query: single query that is send to the Cisco EoX API :raises CiscoApiCallFailed: exception raised if Cisco EoX API call failed :return: list that contains all EoX records from the Cisco EoX API """ if type(api_query) is not str: raise ValueError("api_query must be a string value") # load application settings and check, that the API is enabled app_settings = AppSettings() if not app_settings.is_cisco_api_enabled(): msg = "Cisco API access not enabled" logger.warning(msg) raise CiscoApiCallFailed(msg) # start Cisco EoX API query logger.info("send query to Cisco EoX database: %s" % api_query) eoxapi = CiscoEoxApi() eoxapi.load_client_credentials() results = [] try: current_page = 1 result_pages = 999 while current_page <= result_pages: if current_page == 1: logger.info("Executing API query '%s' on first page" % api_query) else: logger.info("Executing API query '%s' on page '%d" % (api_query, current_page)) # will raise a CiscoApiCallFailed exception on error eoxapi.query_product(product_id=api_query, page=current_page) result_pages = eoxapi.amount_of_pages() if eoxapi.get_page_record_count() > 0: results.extend(eoxapi.get_eox_records()) current_page += 1 except ConnectionFailedException: logger.error("Query failed, server not reachable: %s" % api_query, exc_info=True) raise except CiscoApiCallFailed: logger.fatal("Query failed: %s" % api_query, exc_info=True) raise return results
def test_periodic_sync_enabled(self): # create new AppSettings object and create defaults settings = AppSettings() assert settings.is_periodic_sync_enabled() is False settings.set_periodic_sync_enabled(True) assert settings.is_periodic_sync_enabled() is True settings.set_periodic_sync_enabled(False) assert settings.is_periodic_sync_enabled() is False
def test_cisco_api_enabled(self): # create new AppSettings object and create defaults settings = AppSettings() assert settings.is_cisco_api_enabled() is False settings.set_cisco_api_enabled(True) assert settings.is_cisco_api_enabled() is True settings.set_cisco_api_enabled(False) assert settings.is_cisco_api_enabled() is False
def test_write_datetime_to_settings_file(self): settings = AppSettings() # get a configuration value (default in the global section) now = datetime.now() value = now.isoformat() settings.set_cisco_eox_api_auto_sync_last_execution_time(value) read_value = settings.get_cisco_eox_api_auto_sync_last_execution_time() assert value == read_value assert now == parse_datetime(read_value)
def use_test_api_configuration(): app = AppSettings() with open(CREDENTIALS_FILE) as f: content = json.loads(f.read()) app.set_cisco_api_enabled(True) app.set_product_blacklist_regex("") app.set_cisco_eox_api_queries("") app.set_auto_create_new_products(True) app.set_periodic_sync_enabled(False) app.set_cisco_api_client_id(content.get("client_id", "dummy_id")) app.set_cisco_api_client_id(content.get("client_secret", "dummy_secret"))
def use_test_api_configuration(): app = AppSettings() app.set_cisco_api_enabled(True) app.set_product_blacklist_regex("") app.set_cisco_eox_api_queries("") app.set_auto_create_new_products(True) app.set_periodic_sync_enabled(False) app.set_cisco_api_client_id( os.getenv("TEST_CISCO_API_CLIENT_ID", "dummy_id")) app.set_cisco_api_client_secret( os.getenv("TEST_CISCO_API_CLIENT_SECRET", "dummy_secret"))
def test_login_only_mode_configuration(self): # create new AppSettings object and create defaults settings = AppSettings() assert settings.is_login_only_mode() is False settings.set_login_only_mode(True) assert settings.is_login_only_mode() is True settings.set_login_only_mode(False) assert settings.is_login_only_mode() is False
def test_auto_create_new_products(self): settings = AppSettings() # get values value = settings.is_auto_create_new_products() assert value is False # set values settings.set_auto_create_new_products(True) value = settings.is_auto_create_new_products() assert value is True
def mock_cisco_eox_api_access_broken(monkeypatch): app = AppSettings() app.set_cisco_api_enabled(True) app.set_cisco_api_client_id("client_id") app.set_cisco_api_client_id("client_secret") app.set_periodic_sync_enabled(True) app.set_cisco_eox_api_queries("") app.set_product_blacklist_regex("") app.set_auto_create_new_products(True) monkeypatch.setattr(utils, "check_cisco_eox_api_access", lambda client_id, client_secret, drop_credentials=False: False)
def cisco_eox_populate_product_lc_state_sync_field(): """ Periodic job to populate the lc_state_sync field in the Products, which shows that the product lifecycle data are automatically synchronized against the Cisco EoX API in this case :return: """ try: cis_vendor = Vendor.objects.get(name__istartswith="Cisco") except: # Vendor doesn't exist, no steps required logger.fatal( "Vendor \"Cisco Systems\" not found in database, please check your installation" ) return {"error": "Vendor \"Cisco Systems\" not found in database"} cisco_products = Product.objects.filter(vendor=cis_vendor) if cisco_products.count() != 0: app_config = AppSettings() queries = app_config.get_cisco_eox_api_queries_as_list() # escape the query strings queries = [re.escape(e) for e in queries] # convert the wildcard values queries = [e.replace("\\*", ".*") for e in queries] queries = ["^" + e + "$" for e in queries] with transaction.atomic(): # reset all entries for the vendor pl = Product.objects.filter(vendor=cis_vendor) for p in pl: p.lc_state_sync = False p.save() # only set the state sync to true if the periodic synchronization is enabled if app_config.is_periodic_sync_enabled(): for query in queries: pl = Product.objects.filter(product_id__regex=query, vendor=cis_vendor) for p in pl: p.lc_state_sync = True p.save() return {"status": "Database updated"} else: return { "error": "No Products associated to \"Cisco Systems\" found in database" }
def status(request): """ Status page for the Product Database """ app_config = AppSettings() is_cisco_api_enabled = app_config.is_cisco_api_enabled() context = { "is_cisco_api_enabled": is_cisco_api_enabled } if is_cisco_api_enabled: # test access (once every 30 minutes) cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False) # defaults, overwritten if an exception is thrown cisco_eox_api_available = True cisco_eox_api_message = "successful connected to the Cisco EoX API" if not cisco_eox_api_test_successful: try: result = utils.check_cisco_eox_api_access(client_id=app_config.get_cisco_api_client_id(), client_secret=app_config.get_cisco_api_client_secret(), drop_credentials=False) cache.set("CISCO_EOX_API_TEST", result, 60 * 30) except Exception as ex: cisco_eox_api_available = True cisco_eox_api_message = str(ex) context["cisco_eox_api_available"] = cisco_eox_api_available context["cisco_eox_api_message"] = cisco_eox_api_message # determine worker status state = celery.is_worker_active() if state and not settings.DEBUG: worker_status = """ <div class="alert alert-success" role="alert"> <span class="fa fa-info-circle"></span> Backend worker found. </div>""" else: worker_status = """ <div class="alert alert-danger" role="alert"> <span class="fa fa-exclamation-circle"></span> No backend worker found, asynchronous and scheduled tasks are not executed. </div>""" context['worker_status'] = mark_safe(worker_status) return render(request, "config/status.html", context=context)
def test_product_blacklist_regex_from_config(self): settings = AppSettings() # get values value = settings.get_product_blacklist_regex() assert value == "" # set values blacklist_entries = "test\nanother\ntest" settings.set_product_blacklist_regex(blacklist_entries) value = settings.get_product_blacklist_regex() assert value == blacklist_entries
def test_cisco_api_client_secret(self): settings = AppSettings() # get values value = settings.get_cisco_api_client_secret() assert value == "PlsChgMe" # set values test_client_secret = "test_secret" settings.set_cisco_api_client_secret(test_client_secret) value = settings.get_cisco_api_client_secret() assert value == test_client_secret
def test_internal_product_id_label(self): settings = AppSettings() # get values value = settings.get_internal_product_id_label() assert value == "Internal Product ID" # set values test_internal_product_label = "My custom label" settings.set_internal_product_id_label(test_internal_product_label) value = settings.get_internal_product_id_label() assert value == test_internal_product_label
def test_cisco_eox_wait_time(self): settings = AppSettings() # get value value = settings.get_cisco_eox_api_sync_wait_time() assert value == "5" # set values test_cisco_eox_wait_time = "10" settings.set_cisco_eox_api_sync_wait_time(test_cisco_eox_wait_time) value = settings.get_cisco_eox_api_sync_wait_time() assert value == test_cisco_eox_wait_time
def test_cisco_api_client_id(self): settings = AppSettings() # get values value = settings.get_cisco_api_client_id() assert value == "PlsChgMe" # set values test_client_id = "test_id" settings.set_cisco_api_client_id(test_client_id) value = settings.get_cisco_api_client_id() assert value == test_client_id
def test_cisco_eox_api_auto_sync_last_execution_result(self): settings = AppSettings() # get values value = settings.get_cisco_eox_api_auto_sync_last_execution_result() assert value is None # set values test_last_execution_result = "everything is fine" settings.set_cisco_eox_api_auto_sync_last_execution_result( test_last_execution_result) value = settings.get_cisco_eox_api_auto_sync_last_execution_result() assert value == test_last_execution_result
def update_cisco_eox_records(records): """ update given database records from the Cisco EoX v5 API :param records: :return: """ app_config = AppSettings() blacklist_raw_string = app_config.get_product_blacklist_regex() create_missing = app_config.is_auto_create_new_products() # build blacklist from configuration blacklist = [] for e in [e.split(";") for e in blacklist_raw_string.splitlines()]: blacklist += e blacklist = [e for e in blacklist if e != ""] counter = 0 messages = {} for record in records: blacklisted = False for regex in blacklist: try: if re.search(regex, record["EOLProductID"], re.I): blacklisted = True break except: logger.warning("invalid regular expression in blacklist: %s" % regex) if not blacklisted: try: message = cisco_eox_api_crawler.update_local_db_based_on_record(record, create_missing) if message: messages[record["EOLProductID"]] = message except ValidationError as ex: logger.error("invalid data received from Cisco API, cannot save data object for " "'%s' (%s)" % (record, str(ex)), exc_info=True) else: messages[record["EOLProductID"]] = " Product record ignored" counter += 1 return { "count": counter, "messages": messages }
def test_manual_task(self, monkeypatch): self.mock_api_call(monkeypatch) # try to execute it, while no auto-sync is enabled app = AppSettings() app.set_periodic_sync_enabled(False) app.set_cisco_eox_api_queries("WS-C2960-*") task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True) expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>The following products are ' \ 'affected by this update:</p><ul><li>create the Product <code>WS-C2950G-48-EI-WS</code> ' \ 'in the database</li><li>create the Product <code>WS-C2950T-48-SI-WS</code> in the ' \ 'database</li><li>create the Product <code>WS-C2950G-24-EI</code> in the database</li>' \ '</ul></div>' assert task is not None assert task.status == "SUCCESS", task.traceback assert task.state == TaskState.SUCCESS assert task.info.get("status_message") == expected_result assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message" assert Product.objects.count() == 3, "Three products are part of the update" # test no changes required task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True) expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>No changes required.</div>' assert task is not None assert task.status == "SUCCESS", task.traceback assert task.state == TaskState.SUCCESS assert task.info.get("status_message") == expected_result assert NotificationMessage.objects.count() == 2, "Task should create a Notification Message" assert Product.objects.count() == 3, "Three products are part of the update" # test update required p = Product.objects.get(product_id="WS-C2950G-24-EI") p.eox_update_time_stamp = datetime.date(1999, 1, 1) p.save() task = tasks.execute_task_to_synchronize_cisco_eox_states.delay(ignore_periodic_sync_flag=True) expected_result = '<div style="text-align:left;"><h3>Query: WS-C2960-*</h3>The following products are ' \ 'affected by this update:</p><ul><li>update the Product data for <code>WS-C2950G-24-EI' \ '</code></li></ul></div>' assert task is not None assert task.status == "SUCCESS", task.traceback assert task.state == TaskState.SUCCESS assert task.info.get("status_message") == expected_result assert NotificationMessage.objects.count() == 3, "Task should create a Notification Message" assert Product.objects.count() == 3, "Three products are part of the update"
def test_periodic_task_without_queries(self, monkeypatch): self.mock_api_call(monkeypatch) # test automatic trigger app = AppSettings() app.set_periodic_sync_enabled(True) app.set_cisco_eox_api_queries("") task = tasks.execute_task_to_synchronize_cisco_eox_states.delay() assert task is not None assert task.status == "SUCCESS", task.traceback assert task.state == TaskState.SUCCESS assert task.info.get("status_message") == "No Cisco EoX API queries configured." assert NotificationMessage.objects.count() == 1, "Task should create a Notification Message"