def test_auto_create_new_products(self): settings = AppSettings() # get values value = settings.is_auto_create_new_products() assert value is False # set values settings.set_auto_create_new_products(True) value = settings.is_auto_create_new_products() assert value is True
def update_cisco_eox_records(records): """ update given database records from the Cisco EoX v5 API :param records: :return: """ app_config = AppSettings() blacklist_raw_string = app_config.get_product_blacklist_regex() create_missing = app_config.is_auto_create_new_products() # build blacklist from configuration blacklist = [] for e in [e.split(";") for e in blacklist_raw_string.splitlines()]: blacklist += e blacklist = [e for e in blacklist if e != ""] counter = 0 messages = {} for record in records: blacklisted = False for regex in blacklist: try: if re.search(regex, record["EOLProductID"], re.I): blacklisted = True break except: logger.warning("invalid regular expression in blacklist: %s" % regex) if not blacklisted: try: message = cisco_eox_api_crawler.update_local_db_based_on_record(record, create_missing) if message: messages[record["EOLProductID"]] = message except ValidationError as ex: logger.error("invalid data received from Cisco API, cannot save data object for " "'%s' (%s)" % (record, str(ex)), exc_info=True) else: messages[record["EOLProductID"]] = " Product record ignored" counter += 1 return { "count": counter, "messages": messages }
def change_configuration(request): """ change configuration of the Product Database """ # read settings from configuration file app_config = AppSettings() # read settings from database hp_content_after, _ = TextBlock.objects.get_or_create( name=TextBlock.TB_HOMEPAGE_TEXT_AFTER_FAVORITE_ACTIONS ) hp_content_before, _ = TextBlock.objects.get_or_create( name=TextBlock.TB_HOMEPAGE_TEXT_BEFORE_FAVORITE_ACTIONS ) if request.method == "POST": # create a form instance and populate it with data from the request: form = SettingsForm(request.POST) if form.is_valid(): # set common settings app_config.set_login_only_mode(form.cleaned_data["login_only_mode"]) hp_content_before.html_content = form.cleaned_data["homepage_text_before"] hp_content_before.save() hp_content_after.html_content = form.cleaned_data["homepage_text_after"] hp_content_after.save() # set the Cisco API configuration options api_enabled = form.cleaned_data["cisco_api_enabled"] if not api_enabled: # api is disabled, reset values to default app_config.set_cisco_api_enabled(api_enabled) app_config.set_cisco_api_client_id("PlsChgMe") app_config.set_cisco_api_client_secret("PlsChgMe") app_config.set_periodic_sync_enabled(False) app_config.set_auto_create_new_products(False) app_config.set_cisco_eox_api_queries("") app_config.set_product_blacklist_regex("") app_config.set_cisco_eox_api_sync_wait_time("5") else: app_config.set_cisco_api_enabled(api_enabled) client_id = form.cleaned_data["cisco_api_client_id"] \ if form.cleaned_data["cisco_api_client_id"] != "" else "PlsChgMe" app_config.set_cisco_api_client_id(client_id) client_secret = form.cleaned_data["cisco_api_client_secret"] \ if form.cleaned_data["cisco_api_client_secret"] != "" else "PlsChgMe" app_config.set_cisco_api_client_secret(client_secret) app_config.set_internal_product_id_label(form.cleaned_data["internal_product_id_label"]) app_config.set_periodic_sync_enabled(form.cleaned_data["eox_api_auto_sync_enabled"]) app_config.set_auto_create_new_products(form.cleaned_data["eox_auto_sync_auto_create_elements"]) app_config.set_cisco_eox_api_queries(form.cleaned_data["eox_api_queries"]) app_config.set_product_blacklist_regex(form.cleaned_data["eox_api_blacklist"]) if form.cleaned_data["eox_api_wait_time"]: app_config.set_cisco_eox_api_sync_wait_time(form.cleaned_data["eox_api_wait_time"]) if client_id != "PlsChgMe": result = utils.check_cisco_eox_api_access( form.cleaned_data["cisco_api_client_id"], form.cleaned_data["cisco_api_client_secret"] ) if result: messages.success(request, "Successfully connected to the Cisco EoX API") else: messages.error(request, "Cannot contact the Cisco EoX API. Please contact your " "Administrator") else: messages.info( request, "Please configure your Cisco API credentials within the Cisco API settings tab." ) # expire cached settings cache.delete("LOGIN_ONLY_MODE_SETTING") messages.success(request, "Settings saved successfully") return redirect(resolve_url("productdb_config:change_settings")) else: messages.error(request, "Invalid configuration option detected, please check it below.") else: form = SettingsForm() form.fields['cisco_api_enabled'].initial = app_config.is_cisco_api_enabled() form.fields['login_only_mode'].initial = app_config.is_login_only_mode() form.fields['internal_product_id_label'].initial = app_config.get_internal_product_id_label() form.fields['cisco_api_client_id'].initial = app_config.get_cisco_api_client_id() form.fields['cisco_api_client_secret'].initial = app_config.get_cisco_api_client_secret() form.fields['eox_api_auto_sync_enabled'].initial = app_config.is_periodic_sync_enabled() form.fields['eox_auto_sync_auto_create_elements'].initial = app_config.is_auto_create_new_products() form.fields['eox_api_queries'].initial = app_config.get_cisco_eox_api_queries() form.fields['eox_api_blacklist'].initial = app_config.get_product_blacklist_regex() form.fields['eox_api_wait_time'].initial = app_config.get_cisco_eox_api_sync_wait_time() form.fields['homepage_text_before'].initial = hp_content_before.html_content form.fields['homepage_text_after'].initial = hp_content_after.html_content context = { "form": form, "is_cisco_api_enabled": app_config.is_cisco_api_enabled(), "is_cisco_eox_api_auto_sync_enabled": app_config.is_periodic_sync_enabled() } return render(request, "config/change_configuration.html", context=context)
def update_cisco_eox_database(api_query): """ synchronizes the local database with the Cisco EoX API using the specified query :param api_query: single query that is send to the Cisco EoX API :raises CiscoApiCallFailed: exception raised if Cisco EoX API call failed :return: list of dictionary that describe the updates to the database """ if type(api_query) is not str: raise ValueError("api_query must be a string value") # load application settings and check, that the API is enabled app_settings = AppSettings() if not app_settings.is_cisco_api_enabled(): msg = "Cisco API access not enabled" logger.warn(msg) raise CiscoApiCallFailed(msg) blacklist_raw_string = app_settings.get_product_blacklist_regex() create_missing = app_settings.is_auto_create_new_products() # clean blacklist string and remove empty statements # (split lines, if any and split the string by semicolon) blacklist = [] for e in [e.split(";") for e in blacklist_raw_string.splitlines()]: blacklist += e blacklist = [e for e in blacklist if e != ""] # start Cisco EoX API query logger.info("Query EoX database: %s" % api_query) eoxapi = CiscoEoxApi() eoxapi.load_client_credentials() results = [] try: max_pages = 999 current_page = 1 result_pages = 0 while current_page <= max_pages: logger.info("Executing API query '%s' on page '%d" % (api_query, current_page)) # will raise a CiscoApiCallFailed exception on error eoxapi.query_product(product_id=api_query, page=current_page) if current_page == 1: result_pages = eoxapi.amount_of_pages() logger.info("Initial query returns %d page(s)" % result_pages) records = eoxapi.get_eox_records() # check that the query has valid results if eoxapi.get_page_record_count() > 0: # processing records for record in records: result_record = {} pid = record["EOLProductID"] result_record["PID"] = pid result_record["created"] = False result_record["updated"] = False result_record["message"] = None logger.info("processing product '%s'..." % pid) pid_in_database = product_id_in_database(pid) # check if the product ID is blacklisted by a regular expression pid_blacklisted = False for regex in blacklist: try: if re.search(regex, pid, re.I): pid_blacklisted = True break except: # silently ignore the issue, invalid regular expressions are handled by the settings form logger.info("invalid regular expression: %s" % regex) # ignore if the product id is not in the database if pid_blacklisted and not pid_in_database: logger.info("Product '%s' blacklisted... no further processing" % pid) result_record.update({"blacklist": True}) else: res = {} try: res = update_local_db_based_on_record(record, create_missing) except ValidationError: logger.error( "invalid data received from Cisco API, cannot save data object for '%s'" % pid, exc_info=True, ) result_record["message"] = "invalid data received from Cisco EoX API, import incomplete" finally: res["blacklist"] = False result_record.update(res) results.append(result_record) if current_page == result_pages: break else: current_page += 1 # filter empty result strings if len(results) == 0: results = [ { "PID": None, "blacklist": False, "updated": False, "created": False, "message": "No product update required", } ] except ConnectionFailedException: logger.error("Query failed, server not reachable: %s" % api_query, exc_info=True) raise except CiscoApiCallFailed: logger.fatal("Query failed: %s" % api_query, exc_info=True) raise return results
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False): """ This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the results in the local database. There are two types of operation: * cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the blacklist and not in the database * cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already included in the database :return: """ app_config = AppSettings() run_task = app_config.is_periodic_sync_enabled() if run_task or ignore_periodic_sync_flag: logger.info("start sync with Cisco EoX API...") self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "sync with Cisco EoX API..." }) # read configuration for the Cisco EoX API synchronization queries = app_config.get_cisco_eox_api_queries_as_list() blacklist_raw_string = app_config.get_product_blacklist_regex() create_missing = app_config.is_auto_create_new_products() if len(queries) == 0: result = { "status_message": "No Cisco EoX API queries configured." } NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_WARNING, summary_message="There are no Cisco EoX API queries configured. Nothing to do.", detailed_message="There are no Cisco EoX API queries configured. Please configure at least on EoX API " "query in the settings or disable the periodic synchronization." ) # update the local database with the Cisco EoX API else: # test Cisco EoX API access test_result = utils.check_cisco_eox_api_access( app_config.get_cisco_api_client_id(), app_config.get_cisco_api_client_secret(), False ) if test_result: # execute all queries from the configuration query_eox_records = {} failed_queries = [] failed_query_msgs = {} successful_queries = [] counter = 1 for query in queries: self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of " "%d</strong>)..." % (query, counter, len(queries)) }) # wait some time between the query calls time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time())) try: query_eox_records[query] = cisco_eox_api_crawler.get_raw_api_data(api_query=query) successful_queries.append(query) except CiscoApiCallFailed as ex: msg = "Cisco EoX API call failed (%s)" % str(ex) logger.error("Query %s to Cisco EoX API failed (%s)" % (query, msg), exc_info=True) failed_queries.append(query) failed_query_msgs[query] = str(ex) except Exception as ex: msg = "Unexpected Exception, cannot access the Cisco API. Please ensure that the server is " \ "connected to the internet and that the authentication settings are " \ "valid." logger.error("Query %s to Cisco EoX API failed (%s)" % (query, msg), exc_info=True) failed_queries.append(query) failed_query_msgs[query] = str(ex) counter += 1 # build blacklist from configuration blacklist = [] for e in [e.split(";") for e in blacklist_raw_string.splitlines()]: blacklist += e blacklist = [e for e in blacklist if e != ""] # update data in database self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "update database..." }) messages = {} for key in query_eox_records: amount_of_records = len(query_eox_records[key]) self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "update database (query <code>%s</code>, processed <b>0</b> of " "<b>%d</b> results)..." % (key, amount_of_records) }) counter = 0 for record in query_eox_records[key]: if counter % 100 == 0: self.update_state(state=TaskState.PROCESSING, meta={ "status_message": "update database (query <code>%s</code>, processed <b>%d</b> of " "<b>%d</b> results)..." % (key, counter, amount_of_records) }) blacklisted = False for regex in blacklist: try: if re.search(regex, record["EOLProductID"], re.I): blacklisted = True break except: logger.warning("invalid regular expression in blacklist: %s" % regex) if not blacklisted: try: message = cisco_eox_api_crawler.update_local_db_based_on_record(record, create_missing) if message: messages[record["EOLProductID"]] = message except ValidationError as ex: logger.error("invalid data received from Cisco API, cannot save data object for " "'%s' (%s)" % (record, str(ex)), exc_info=True) else: messages[record["EOLProductID"]] = " Product record ignored" counter += 1 # view the queries in the detailed message and all messages (if there are some) detailed_message = "The following queries were executed:<br><ul style=\"text-align: left;\">" for fq in failed_queries: detailed_message += "<li class=\"text-danger\"><code>%s</code> " \ "(failed, %s)</li>" % (fq, failed_query_msgs.get(fq, "unknown")) for sq in successful_queries: detailed_message += "<li><code>%s</code> (<b>affects %d products</b>, " \ "success)</li>" % (sq, len(query_eox_records[sq])) detailed_message += "</ul>" if len(messages) > 0: detailed_message += "<br>The following comment/errors occurred " \ "during the synchronization:<br><ul style=\"text-align: left;\">" for e in messages.keys(): detailed_message += "<li><code>%s</code>: %s</li>" % (e, messages[e]) detailed_message += "</ul>" # show the executed queries in the summary message if len(failed_queries) == 0 and len(successful_queries) != 0: summary_html = "The following queries were successful executed: %s" % ", ".join( ["<code>%s</code>" % query for query in successful_queries] ) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_SUCCESS, summary_message="The synchronization with the Cisco EoX API was successful. " + summary_html, detailed_message=detailed_message ) elif len(failed_queries) != 0 and len(successful_queries) == 0: summary_html = "The following queries failed to execute: %s" % ", ".join( ["<code>%s</code>" % query for query in failed_queries] ) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR, summary_message="The synchronization with the Cisco EoX API was not successful. " + summary_html, detailed_message=detailed_message ) else: summary_html = "The following queries were successful executed: %s\n<br>The following queries " \ "failed to execute: %s" % ( ", ".join(["<code>%s</code>" % query for query in successful_queries]), ", ".join(["<code>%s</code>" % query for query in failed_queries]) ) NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_WARNING, summary_message="The synchronization with the Cisco EoX API was partially " "successful. " + summary_html, detailed_message=detailed_message ) result = {"status_message": "<p style=\"text-align: left;\">" + detailed_message + "</p>"} # if the task was executed eager, set state to SUCCESS (required for testing) if self.request.is_eager: self.update_state(state=TaskState.SUCCESS, meta={"status_message": summary_html}) else: msg = "Cannot contact Cisco EoX API, please verify your internet connection and access " \ "credentials." if not ignore_periodic_sync_flag: NotificationMessage.objects.create( title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR, summary_message="The synchronization with the Cisco EoX API was not successful.", detailed_message=msg ) result = { "error_message": msg } else: result = { "status_message": "task not enabled" } # remove in progress flag with the cache cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS") return result
def update_cisco_eox_database(api_query): """ synchronizes the local database with the Cisco EoX API using the specified query :param api_query: single query that is send to the Cisco EoX API :raises CiscoApiCallFailed: exception raised if Cisco EoX API call failed :return: list of dictionary that describe the updates to the database """ if type(api_query) is not str: raise ValueError("api_query must be a string value") # load application settings and check, that the API is enabled app_settings = AppSettings() if not app_settings.is_cisco_api_enabled(): msg = "Cisco API access not enabled" logger.warn(msg) raise CiscoApiCallFailed(msg) blacklist_raw_string = app_settings.get_product_blacklist_regex() create_missing = app_settings.is_auto_create_new_products() # clean blacklist string and remove empty statements # (split lines, if any and split the string by semicolon) blacklist = [] for e in [e.split(";") for e in blacklist_raw_string.splitlines()]: blacklist += e blacklist = [e for e in blacklist if e != ""] # start Cisco EoX API query logger.info("Query EoX database: %s" % api_query) eoxapi = CiscoEoxApi() eoxapi.load_client_credentials() results = [] try: max_pages = 999 current_page = 1 result_pages = 0 while current_page <= max_pages: logger.info("Executing API query '%s' on page '%d" % (api_query, current_page)) # will raise a CiscoApiCallFailed exception on error eoxapi.query_product(product_id=api_query, page=current_page) if current_page == 1: result_pages = eoxapi.amount_of_pages() logger.info("Initial query returns %d page(s)" % result_pages) records = eoxapi.get_eox_records() # check that the query has valid results if eoxapi.get_page_record_count() > 0: # processing records for record in records: result_record = {} pid = record['EOLProductID'] result_record["PID"] = pid result_record["created"] = False result_record["updated"] = False result_record["message"] = None logger.info("processing product '%s'..." % pid) pid_in_database = product_id_in_database(pid) # check if the product ID is blacklisted by a regular expression pid_blacklisted = False for regex in blacklist: try: if re.search(regex, pid, re.I): pid_blacklisted = True break except: # silently ignore the issue, invalid regular expressions are handled by the settings form logger.info("invalid regular expression: %s" % regex) # ignore if the product id is not in the database if pid_blacklisted and not pid_in_database: logger.info("Product '%s' blacklisted... no further processing" % pid) result_record.update({ "blacklist": True }) else: res = {} try: res = update_local_db_based_on_record(record, create_missing) except ValidationError: logger.error("invalid data received from Cisco API, cannot save data object for '%s'" % pid, exc_info=True) result_record["message"] = "invalid data received from Cisco EoX API, import incomplete" finally: res["blacklist"] = False result_record.update(res) results.append(result_record) if current_page == result_pages: break else: current_page += 1 # filter empty result strings if len(results) == 0: results = [ { "PID": None, "blacklist": False, "updated": False, "created": False, "message": "No product update required" } ] except ConnectionFailedException: logger.error("Query failed, server not reachable: %s" % api_query, exc_info=True) raise except CiscoApiCallFailed: logger.fatal("Query failed: %s" % api_query, exc_info=True) raise return results
def change_configuration(request): """ change configuration of the Product Database """ # read settings from configuration file app_config = AppSettings() # read settings from database hp_content_after, _ = TextBlock.objects.get_or_create( name=TextBlock.TB_HOMEPAGE_TEXT_AFTER_FAVORITE_ACTIONS ) hp_content_before, _ = TextBlock.objects.get_or_create( name=TextBlock.TB_HOMEPAGE_TEXT_BEFORE_FAVORITE_ACTIONS ) if request.method == "POST": # create a form instance and populate it with data from the request: form = SettingsForm(request.POST) if form.is_valid(): # set common settings app_config.set_login_only_mode(form.cleaned_data["login_only_mode"]) hp_content_before.html_content = form.cleaned_data["homepage_text_before"] hp_content_before.save() hp_content_after.html_content = form.cleaned_data["homepage_text_after"] hp_content_after.save() # set the Cisco API configuration options api_enabled = form.cleaned_data["cisco_api_enabled"] if not api_enabled: # api is disabled, reset values to default app_config.set_cisco_api_enabled(api_enabled) app_config.set_cisco_api_client_id("PlsChgMe") app_config.set_cisco_api_client_secret("PlsChgMe") app_config.set_periodic_sync_enabled(False) app_config.set_auto_create_new_products(False) app_config.set_cisco_eox_api_queries("") app_config.set_product_blacklist_regex("") app_config.set_cisco_eox_api_sync_wait_time("5") else: app_config.set_cisco_api_enabled(api_enabled) client_id = form.cleaned_data["cisco_api_client_id"] \ if form.cleaned_data["cisco_api_client_id"] != "" else "PlsChgMe" app_config.set_cisco_api_client_id(client_id) client_secret = form.cleaned_data["cisco_api_client_secret"] \ if form.cleaned_data["cisco_api_client_secret"] != "" else "PlsChgMe" app_config.set_cisco_api_client_secret(client_secret) if client_id != "PlsChgMe": result = utils.check_cisco_eox_api_access( form.cleaned_data["cisco_api_client_id"], form.cleaned_data["cisco_api_client_secret"] ) if result: messages.success(request, "Successfully connected to the Cisco EoX API") else: messages.error(request, "Cannot contact the Cisco EoX API. Please contact your " "Administrator") else: messages.info( request, "Please configure your Cisco API credentials within the Cisco API settings tab." ) app_config.set_periodic_sync_enabled(form.cleaned_data["eox_api_auto_sync_enabled"]) app_config.set_internal_product_id_label(form.cleaned_data["internal_product_id_label"]) app_config.set_auto_create_new_products(form.cleaned_data["eox_auto_sync_auto_create_elements"]) app_config.set_cisco_eox_api_queries(form.cleaned_data["eox_api_queries"]) app_config.set_product_blacklist_regex(form.cleaned_data["eox_api_blacklist"]) if form.cleaned_data["eox_api_wait_time"]: app_config.set_cisco_eox_api_sync_wait_time(form.cleaned_data["eox_api_wait_time"]) # expire cached settings cache.delete("LOGIN_ONLY_MODE_SETTING") messages.success(request, "Settings saved successfully") return redirect(resolve_url("productdb_config:change_settings")) else: messages.error(request, "Invalid configuration option detected, please check it below.") else: form = SettingsForm() form.fields['cisco_api_enabled'].initial = app_config.is_cisco_api_enabled() form.fields['login_only_mode'].initial = app_config.is_login_only_mode() form.fields['internal_product_id_label'].initial = app_config.get_internal_product_id_label() form.fields['cisco_api_client_id'].initial = app_config.get_cisco_api_client_id() form.fields['cisco_api_client_secret'].initial = app_config.get_cisco_api_client_secret() form.fields['eox_api_auto_sync_enabled'].initial = app_config.is_periodic_sync_enabled() form.fields['eox_auto_sync_auto_create_elements'].initial = app_config.is_auto_create_new_products() form.fields['eox_api_queries'].initial = app_config.get_cisco_eox_api_queries() form.fields['eox_api_blacklist'].initial = app_config.get_product_blacklist_regex() form.fields['eox_api_wait_time'].initial = app_config.get_cisco_eox_api_sync_wait_time() form.fields['homepage_text_before'].initial = hp_content_before.html_content form.fields['homepage_text_after'].initial = hp_content_after.html_content context = { "form": form, "is_cisco_api_enabled": app_config.is_cisco_api_enabled(), "is_cisco_eox_api_auto_sync_enabled": app_config.is_periodic_sync_enabled() } return render(request, "config/change_configuration.html", context=context)