def test_cisco_api_client_id(self):
        settings = AppSettings()

        # get values
        value = settings.get_cisco_api_client_id()
        assert value == "PlsChgMe"

        # set values
        test_client_id = "test_id"
        settings.set_cisco_api_client_id(test_client_id)
        value = settings.get_cisco_api_client_id()

        assert value == test_client_id
    def test_cisco_api_client_id(self):
        settings = AppSettings()

        # get values
        value = settings.get_cisco_api_client_id()
        assert value == "PlsChgMe"

        # set values
        test_client_id = "test_id"
        settings.set_cisco_api_client_id(test_client_id)
        value = settings.get_cisco_api_client_id()

        assert value == test_client_id
Exemplo n.º 3
0
    def load_client_credentials(self):
        logger.debug("load client credentials from configuration")
        app_settings = AppSettings()

        # load client credentials
        self.client_id = app_settings.get_cisco_api_client_id()
        self.client_secret = app_settings.get_cisco_api_client_secret()
Exemplo n.º 4
0
    def load_client_credentials(self):
        logger.debug("load client credentials from configuration")
        app_settings = AppSettings()

        # load client credentials
        self.client_id = app_settings.get_cisco_api_client_id()
        self.client_secret = app_settings.get_cisco_api_client_secret()
Exemplo n.º 5
0
def status(request):
    """
    Status page for the Product Database
    """
    app_config = AppSettings()

    is_cisco_api_enabled = app_config.is_cisco_api_enabled()
    context = {
        "is_cisco_api_enabled": is_cisco_api_enabled
    }

    if is_cisco_api_enabled:
        # test access (once every 30 minutes)
        cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)

        # defaults, overwritten if an exception is thrown
        cisco_eox_api_available = True
        cisco_eox_api_message = "successful connected to the Cisco EoX API"

        if not cisco_eox_api_test_successful:
            try:
                result = utils.check_cisco_eox_api_access(client_id=app_config.get_cisco_api_client_id(),
                                                          client_secret=app_config.get_cisco_api_client_secret(),
                                                          drop_credentials=False)
                cache.set("CISCO_EOX_API_TEST", result, 60 * 30)

            except Exception as ex:
                cisco_eox_api_available = True
                cisco_eox_api_message = str(ex)

        context["cisco_eox_api_available"] = cisco_eox_api_available
        context["cisco_eox_api_message"] = cisco_eox_api_message

    # determine worker status
    state = celery.is_worker_active()

    if state and not settings.DEBUG:
        worker_status = """
            <div class="alert alert-success" role="alert">
                <span class="fa fa-info-circle"></span>
                Backend worker found.
            </div>"""

    else:
        worker_status = """
            <div class="alert alert-danger" role="alert">
                <span class="fa fa-exclamation-circle"></span>
                No backend worker found, asynchronous and scheduled tasks are not executed.
            </div>"""

    context['worker_status'] = mark_safe(worker_status)

    return render(request, "config/status.html", context=context)
Exemplo n.º 6
0
def status(request):
    """
    Status page for the Product Database
    """
    app_config = AppSettings()

    is_cisco_api_enabled = app_config.is_cisco_api_enabled()
    context = {"is_cisco_api_enabled": is_cisco_api_enabled}

    if is_cisco_api_enabled:
        # test access (once every 30 minutes)
        cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)

        # defaults, overwritten if an exception is thrown
        cisco_eox_api_available = True
        cisco_eox_api_message = "successful connected to the Cisco EoX API"

        if not cisco_eox_api_test_successful:
            try:
                result = utils.check_cisco_eox_api_access(
                    client_id=app_config.get_cisco_api_client_id(),
                    client_secret=app_config.get_cisco_api_client_secret(),
                    drop_credentials=False)
                cache.set("CISCO_EOX_API_TEST", result, 60 * 30)

            except Exception as ex:
                cisco_eox_api_available = True
                cisco_eox_api_message = str(ex)

        context["cisco_eox_api_available"] = cisco_eox_api_available
        context["cisco_eox_api_message"] = cisco_eox_api_message

    # determine worker status
    context['worker_status'] = mark_safe(utils.get_celery_worker_state_html())

    return render(request, "config/status.html", context=context)
Exemplo n.º 7
0
def execute_task_to_synchronize_cisco_eox_states(
        self, ignore_periodic_sync_flag=False):
    """
    This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
    results in the local database. There are two types of operation:
      * cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
                                                                      blacklist and not in the database
      * cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
                                                                       included in the database

    :return:
    """
    app_config = AppSettings()
    run_task = app_config.is_periodic_sync_enabled()

    if run_task or ignore_periodic_sync_flag:
        logger.info("start sync with Cisco EoX API...")
        self.update_state(
            state=TaskState.PROCESSING,
            meta={"status_message": "sync with Cisco EoX API..."})

        # read queries from configuration
        queries = app_config.get_cisco_eox_api_queries_as_list()

        if len(queries) == 0:
            result = {"status_message": "No Cisco EoX API queries configured."}

            NotificationMessage.objects.create(
                title=NOTIFICATION_MESSAGE_TITLE,
                type=NotificationMessage.MESSAGE_WARNING,
                summary_message=
                "There are no Cisco EoX API queries configured. Nothing to do.",
                detailed_message=
                "There are no Cisco EoX API queries configured. Please configure at least on EoX API "
                "query in the settings or disable the periodic synchronization."
            )

        # update the local database with the Cisco EoX API
        else:
            try:
                # test Cisco EoX API access
                success = utils.check_cisco_eox_api_access(
                    app_config.get_cisco_api_client_id(),
                    app_config.get_cisco_api_client_secret(), False)

                if not success:
                    msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \
                          "and that the authentication settings are valid."
                    logger.error(msg, exc_info=True)

                    NotificationMessage.objects.create(
                        title=NOTIFICATION_MESSAGE_TITLE,
                        type=NotificationMessage.MESSAGE_ERROR,
                        summary_message=msg,
                        detailed_message=
                        "The synchronization with the Cisco EoX API was not started."
                    )

                    result = {"error_message": msg}

                else:
                    # execute all queries from the configuration and collect the results
                    notify_metrics = {"queries": {}}
                    counter = 0
                    for query in queries:
                        self.update_state(
                            state=TaskState.PROCESSING,
                            meta={
                                "status_message":
                                "send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
                                "%d</strong>)..." %
                                (query, counter + 1, len(queries))
                            })

                        # wait a specific amount of seconds between each update call
                        time.sleep(
                            int(app_config.get_cisco_eox_api_sync_wait_time()))

                        query_results = cisco_eox_api_crawler.update_cisco_eox_database(
                            query)

                        blist_counter = 0
                        update_counter = 0
                        create_counter = 0
                        for qres in query_results:
                            if qres["created"]:
                                create_counter += 1
                            elif qres["updated"]:
                                update_counter += 1
                            elif qres["blacklist"]:
                                blist_counter += 1

                        notify_metrics["queries"][query] = {
                            "amount": len(query_results),
                            "updated_entries": update_counter,
                            "created_entries": create_counter,
                            "blacklisted_entries": blist_counter,
                            "result_details": query_results
                        }

                        counter += 1

                    # create NotificationMessage based on the results
                    detailed_html = ""
                    blist_counter = 0
                    update_counter = 0
                    create_counter = 0
                    for query_key in notify_metrics["queries"].keys():
                        update_counter += notify_metrics["queries"][query_key][
                            "updated_entries"]
                        create_counter += notify_metrics["queries"][query_key][
                            "created_entries"]
                        blist_counter += notify_metrics["queries"][query_key][
                            "blacklisted_entries"]

                        # build detailed string
                        detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key

                        cond_1 = notify_metrics["queries"][query_key][
                            "updated_entries"] == 0
                        cond_1 = cond_1 and (notify_metrics["queries"]
                                             [query_key]["created_entries"]
                                             == 0)
                        cond_1 = cond_1 and (notify_metrics["queries"]
                                             [query_key]["blacklisted_entries"]
                                             == 0)

                        if cond_1:
                            detailed_html += "No changes required."

                        else:
                            detailed_html += "The following products are affected by this update:</p>"
                            detailed_html += "<ul>"

                            for qres in notify_metrics["queries"][query_key][
                                    "result_details"]:

                                msg = ""
                                if "message" in qres.keys():
                                    if qres["message"]:
                                        msg = qres["message"]

                                if qres["created"]:
                                    detailed_html += "<li>create the Product <code>%s</code> in the database" % (
                                        qres["PID"])
                                    detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"

                                elif qres["updated"]:
                                    detailed_html += "<li>update the Product data for <code>%s</code>" % (
                                        qres["PID"])
                                    detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"

                                elif qres["blacklist"]:
                                    detailed_html += "<li>Product data for <code>%s</code> ignored" % (
                                        qres["PID"])
                                    detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"

                            detailed_html += "</ul>"

                        detailed_html += "</div>"

                    summary_html = "The synchronization was performed successfully. "

                    if update_counter == 1:
                        summary_html += "<strong>%d</strong> product was updated, " % update_counter
                    else:
                        summary_html += "<strong>%d</strong> products are updated, " % update_counter

                    if create_counter == 1:
                        summary_html += "<strong>%s</strong> product was added to the database and " % create_counter
                    else:
                        summary_html += "<strong>%s</strong> products are added to the database and " % create_counter

                    if blist_counter == 1:
                        summary_html += "<strong>%d</strong> product was ignored." % blist_counter
                    else:
                        summary_html += "<strong>%d</strong> products are ignored." % blist_counter

                    # show the executed queries in the summary message
                    summary_html += " The following queries were executed: %s" % ", ".join(
                        ["<code>%s</code>" % query for query in queries])

                    NotificationMessage.objects.create(
                        title=NOTIFICATION_MESSAGE_TITLE,
                        type=NotificationMessage.MESSAGE_SUCCESS,
                        summary_message=summary_html,
                        detailed_message=detailed_html)

                    result = {"status_message": detailed_html}

                    # if the task was executed eager, set state to SUCCESS (required for testing)
                    if self.request.is_eager:
                        self.update_state(
                            state=TaskState.SUCCESS,
                            meta={"status_message": detailed_html})

            except CredentialsNotFoundException as ex:
                msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(
                    ex)
                logger.error(msg, exc_info=True)

                NotificationMessage.objects.create(
                    title=NOTIFICATION_MESSAGE_TITLE,
                    type=NotificationMessage.MESSAGE_ERROR,
                    summary_message=msg,
                    detailed_message=
                    "The synchronization was performed partially.")

                result = {"error_message": msg}

            except CiscoApiCallFailed as ex:
                msg = "Cisco EoX API call failed (%s)" % str(ex)
                logger.error(msg, exc_info=True)

                NotificationMessage.objects.create(
                    title=NOTIFICATION_MESSAGE_TITLE,
                    type=NotificationMessage.MESSAGE_ERROR,
                    summary_message=msg,
                    detailed_message=
                    "The synchronization was performed partially.")

                result = {"error_message": msg}
            except Exception as ex:
                msg = "Cannot access the Cisco API. Please ensure that the server is " \
                      "connected to the internet and that the authentication settings are " \
                      "valid."
                logger.error(msg, exc_info=True)

                NotificationMessage.objects.create(
                    title=NOTIFICATION_MESSAGE_TITLE,
                    type=NotificationMessage.MESSAGE_ERROR,
                    summary_message=msg,
                    detailed_message="%s" % str(ex))

                result = {"error_message": msg}

    else:
        result = {"status_message": "task not enabled"}

    # remove in progress flag with the cache
    cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")

    return result
Exemplo n.º 8
0
def change_configuration(request):
    """
    change configuration of the Product Database
    """
    # read settings from configuration file
    app_config = AppSettings()

    # read settings from database
    hp_content_after, _ = TextBlock.objects.get_or_create(
        name=TextBlock.TB_HOMEPAGE_TEXT_AFTER_FAVORITE_ACTIONS
    )
    hp_content_before, _ = TextBlock.objects.get_or_create(
        name=TextBlock.TB_HOMEPAGE_TEXT_BEFORE_FAVORITE_ACTIONS
    )

    if request.method == "POST":
        # create a form instance and populate it with data from the request:
        form = SettingsForm(request.POST)
        if form.is_valid():
            # set common settings
            app_config.set_login_only_mode(form.cleaned_data["login_only_mode"])

            hp_content_before.html_content = form.cleaned_data["homepage_text_before"]
            hp_content_before.save()
            hp_content_after.html_content = form.cleaned_data["homepage_text_after"]
            hp_content_after.save()

            # set the Cisco API configuration options
            api_enabled = form.cleaned_data["cisco_api_enabled"]

            if not api_enabled:
                # api is disabled, reset values to default
                app_config.set_cisco_api_enabled(api_enabled)
                app_config.set_cisco_api_client_id("PlsChgMe")
                app_config.set_cisco_api_client_secret("PlsChgMe")
                app_config.set_periodic_sync_enabled(False)
                app_config.set_auto_create_new_products(False)
                app_config.set_cisco_eox_api_queries("")
                app_config.set_product_blacklist_regex("")
                app_config.set_cisco_eox_api_sync_wait_time("5")

            else:
                app_config.set_cisco_api_enabled(api_enabled)

                client_id = form.cleaned_data["cisco_api_client_id"] \
                    if form.cleaned_data["cisco_api_client_id"] != "" else "PlsChgMe"
                app_config.set_cisco_api_client_id(client_id)
                client_secret = form.cleaned_data["cisco_api_client_secret"] \
                    if form.cleaned_data["cisco_api_client_secret"] != "" else "PlsChgMe"
                app_config.set_cisco_api_client_secret(client_secret)

                app_config.set_internal_product_id_label(form.cleaned_data["internal_product_id_label"])
                app_config.set_periodic_sync_enabled(form.cleaned_data["eox_api_auto_sync_enabled"])
                app_config.set_auto_create_new_products(form.cleaned_data["eox_auto_sync_auto_create_elements"])
                app_config.set_cisco_eox_api_queries(form.cleaned_data["eox_api_queries"])
                app_config.set_product_blacklist_regex(form.cleaned_data["eox_api_blacklist"])
                if form.cleaned_data["eox_api_wait_time"]:
                    app_config.set_cisco_eox_api_sync_wait_time(form.cleaned_data["eox_api_wait_time"])

                if client_id != "PlsChgMe":
                    result = utils.check_cisco_eox_api_access(
                        form.cleaned_data["cisco_api_client_id"],
                        form.cleaned_data["cisco_api_client_secret"]
                    )

                    if result:
                        messages.success(request, "Successfully connected to the Cisco EoX API")

                    else:
                        messages.error(request, "Cannot contact the Cisco EoX API. Please contact your "
                                                "Administrator")

                else:
                    messages.info(
                        request,
                        "Please configure your Cisco API credentials within the Cisco API settings tab."
                    )

            # expire cached settings
            cache.delete("LOGIN_ONLY_MODE_SETTING")

            messages.success(request, "Settings saved successfully")
            return redirect(resolve_url("productdb_config:change_settings"))

        else:
            messages.error(request, "Invalid configuration option detected, please check it below.")

    else:
        form = SettingsForm()
        form.fields['cisco_api_enabled'].initial = app_config.is_cisco_api_enabled()
        form.fields['login_only_mode'].initial = app_config.is_login_only_mode()
        form.fields['internal_product_id_label'].initial = app_config.get_internal_product_id_label()
        form.fields['cisco_api_client_id'].initial = app_config.get_cisco_api_client_id()
        form.fields['cisco_api_client_secret'].initial = app_config.get_cisco_api_client_secret()
        form.fields['eox_api_auto_sync_enabled'].initial = app_config.is_periodic_sync_enabled()
        form.fields['eox_auto_sync_auto_create_elements'].initial = app_config.is_auto_create_new_products()
        form.fields['eox_api_queries'].initial = app_config.get_cisco_eox_api_queries()
        form.fields['eox_api_blacklist'].initial = app_config.get_product_blacklist_regex()
        form.fields['eox_api_wait_time'].initial = app_config.get_cisco_eox_api_sync_wait_time()
        form.fields['homepage_text_before'].initial = hp_content_before.html_content
        form.fields['homepage_text_after'].initial = hp_content_after.html_content

    context = {
        "form": form,
        "is_cisco_api_enabled": app_config.is_cisco_api_enabled(),
        "is_cisco_eox_api_auto_sync_enabled": app_config.is_periodic_sync_enabled()
    }
    return render(request, "config/change_configuration.html", context=context)
Exemplo n.º 9
0
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False):
    """
    This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
    results in the local database. There are two types of operation:
      * cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
                                                                      blacklist and not in the database
      * cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
                                                                       included in the database

    :return:
    """
    app_config = AppSettings()
    run_task = app_config.is_periodic_sync_enabled()

    if run_task or ignore_periodic_sync_flag:
        logger.info("start sync with Cisco EoX API...")
        self.update_state(state=TaskState.PROCESSING, meta={
            "status_message": "sync with Cisco EoX API..."
        })

        # read queries from configuration
        queries = app_config.get_cisco_eox_api_queries_as_list()

        if len(queries) == 0:
            result = {
                "status_message": "No Cisco EoX API queries configured."
            }

            NotificationMessage.objects.create(
                title=NOTIFICATION_MESSAGE_TITLE,
                type=NotificationMessage.MESSAGE_WARNING,
                summary_message="There are no Cisco EoX API queries configured. Nothing to do.",
                detailed_message="There are no Cisco EoX API queries configured. Please configure at least on EoX API "
                                 "query in the settings or disable the periodic synchronization."
            )

        # update the local database with the Cisco EoX API
        else:
            try:
                # test Cisco EoX API access
                success = utils.check_cisco_eox_api_access(
                    app_config.get_cisco_api_client_id(),
                    app_config.get_cisco_api_client_secret(),
                    False
                )

                if not success:
                    msg = "Cannot access the Cisco API. Please ensure that the server is connected to the internet " \
                          "and that the authentication settings are valid."
                    logger.error(msg, exc_info=True)

                    NotificationMessage.objects.create(
                        title=NOTIFICATION_MESSAGE_TITLE,
                        type=NotificationMessage.MESSAGE_ERROR,
                        summary_message=msg,
                        detailed_message="The synchronization with the Cisco EoX API was not started."
                    )

                    result = {
                        "error_message": msg
                    }

                else:
                    # execute all queries from the configuration and collect the results
                    notify_metrics = {
                        "queries": {}
                    }
                    counter = 0
                    for query in queries:
                        self.update_state(state=TaskState.PROCESSING, meta={
                            "status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
                                              "%d</strong>)..." % (query, counter + 1, len(queries))
                        })

                        # wait a specific amount of seconds between each update call
                        time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time()))

                        query_results = cisco_eox_api_crawler.update_cisco_eox_database(query)

                        blist_counter = 0
                        update_counter = 0
                        create_counter = 0
                        for qres in query_results:
                            if qres["created"]:
                                create_counter += 1
                            elif qres["updated"]:
                                update_counter += 1
                            elif qres["blacklist"]:
                                blist_counter += 1

                        notify_metrics["queries"][query] = {
                            "amount": len(query_results),
                            "updated_entries": update_counter,
                            "created_entries": create_counter,
                            "blacklisted_entries": blist_counter,
                            "result_details": query_results
                        }

                        counter += 1

                    # create NotificationMessage based on the results
                    detailed_html = ""
                    blist_counter = 0
                    update_counter = 0
                    create_counter = 0
                    for query_key in notify_metrics["queries"].keys():
                        update_counter += notify_metrics["queries"][query_key]["updated_entries"]
                        create_counter += notify_metrics["queries"][query_key]["created_entries"]
                        blist_counter += notify_metrics["queries"][query_key]["blacklisted_entries"]

                        # build detailed string
                        detailed_html += "<div style=\"text-align:left;\"><h3>Query: %s</h3>" % query_key

                        cond_1 = notify_metrics["queries"][query_key]["updated_entries"] == 0
                        cond_1 = cond_1 and (notify_metrics["queries"][query_key]["created_entries"] == 0)
                        cond_1 = cond_1 and (notify_metrics["queries"][query_key]["blacklisted_entries"] == 0)

                        if cond_1:
                            detailed_html += "No changes required."

                        else:
                            detailed_html += "The following products are affected by this update:</p>"
                            detailed_html += "<ul>"

                            for qres in notify_metrics["queries"][query_key]["result_details"]:

                                msg = ""
                                if "message" in qres.keys():
                                    if qres["message"]:
                                        msg = qres["message"]

                                if qres["created"]:
                                    detailed_html += "<li>create the Product <code>%s</code> in the database" % (
                                        qres["PID"]
                                    )
                                    detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"

                                elif qres["updated"]:
                                    detailed_html += "<li>update the Product data for <code>%s</code>" % (
                                        qres["PID"]
                                    )
                                    detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"

                                elif qres["blacklist"]:
                                    detailed_html += "<li>Product data for <code>%s</code> ignored" % (
                                        qres["PID"]
                                    )
                                    detailed_html += " (%s)</li>" % msg if msg != "" else "</li>"

                            detailed_html += "</ul>"

                        detailed_html += "</div>"

                    summary_html = "The synchronization was performed successfully. "

                    if update_counter == 1:
                        summary_html += "<strong>%d</strong> product was updated, " % update_counter
                    else:
                        summary_html += "<strong>%d</strong> products are updated, " % update_counter

                    if create_counter == 1:
                        summary_html += "<strong>%s</strong> product was added to the database and " % create_counter
                    else:
                        summary_html += "<strong>%s</strong> products are added to the database and " % create_counter

                    if blist_counter == 1:
                        summary_html += "<strong>%d</strong> product was ignored." % blist_counter
                    else:
                        summary_html += "<strong>%d</strong> products are ignored." % blist_counter

                    # show the executed queries in the summary message
                    summary_html += " The following queries were executed: %s" % ", ".join(
                        ["<code>%s</code>" % query for query in queries]
                    )

                    NotificationMessage.objects.create(
                        title=NOTIFICATION_MESSAGE_TITLE,
                        type=NotificationMessage.MESSAGE_SUCCESS,
                        summary_message=summary_html,
                        detailed_message=detailed_html
                    )

                    result = {
                        "status_message": detailed_html
                    }

                    # if the task was executed eager, set state to SUCCESS (required for testing)
                    if self.request.is_eager:
                        self.update_state(state=TaskState.SUCCESS, meta={
                            "status_message": detailed_html
                        })

            except CredentialsNotFoundException as ex:
                msg = "Invalid credentials for Cisco EoX API or insufficient access rights (%s)" % str(ex)
                logger.error(msg, exc_info=True)

                NotificationMessage.objects.create(
                    title=NOTIFICATION_MESSAGE_TITLE,
                    type=NotificationMessage.MESSAGE_ERROR,
                    summary_message=msg,
                    detailed_message="The synchronization was performed partially."
                )

                result = {
                    "error_message": msg
                }

            except CiscoApiCallFailed as ex:
                msg = "Cisco EoX API call failed (%s)" % str(ex)
                logger.error(msg, exc_info=True)

                NotificationMessage.objects.create(
                    title=NOTIFICATION_MESSAGE_TITLE,
                    type=NotificationMessage.MESSAGE_ERROR,
                    summary_message=msg,
                    detailed_message="The synchronization was performed partially."
                )

                result = {
                    "error_message": msg
                }
            except Exception as ex:
                msg = "Cannot access the Cisco API. Please ensure that the server is " \
                      "connected to the internet and that the authentication settings are " \
                      "valid."
                logger.error(msg, exc_info=True)

                NotificationMessage.objects.create(
                    title=NOTIFICATION_MESSAGE_TITLE,
                    type=NotificationMessage.MESSAGE_ERROR,
                    summary_message=msg,
                    detailed_message="%s" % str(ex)
                )

                result = {
                    "error_message": msg
                }

    else:
        result = {
            "status_message": "task not enabled"
        }

    # remove in progress flag with the cache
    cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")

    return result
Exemplo n.º 10
0
def initial_sync_with_cisco_eox_api(self, years_list):
    """
    synchronize all entries from the EoX API for a given amount of years (today - n-years), ignores the create missing
    entries and the configurable blacklist.
    :param self:
    :param years_list: list of years to sync (e.g. [2018, 2017, 2016]
    :return:
    """
    if type(years_list) is not list:
        raise AttributeError("years_list must be a list")

    for val in years_list:
        if type(val) is not int:
            raise AttributeError("years_list must be a list of integers")

    if len(years_list) == 0:
        return {
            "status_message": "No years provided, nothing to do."
        }

    app_config = AppSettings()

    # test Cisco EoX API access
    test_result = utils.check_cisco_eox_api_access(
        app_config.get_cisco_api_client_id(),
        app_config.get_cisco_api_client_secret(),
        False
    )
    failed_years = []
    successful_years = []

    if test_result:
        # perform synchronization
        self.update_state(state=TaskState.PROCESSING, meta={
            "status_message": "start initial synchronization with the Cisco EoX API..."
        })
        all_records = []
        for year in years_list:
            self.update_state(state=TaskState.PROCESSING, meta={
                "status_message": "fetch all information for year %d..." % year
            })
            # wait some time between the query calls
            time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time()))

            # fetch all API entries for a specific year
            try:
                records = cisco_eox_api_crawler.get_raw_api_data(year=year)
                successful_years += [year]

                all_records.append({
                    "year": year,
                    "records": records
                })

            except CiscoApiCallFailed as ex:
                msg = "Cisco EoX API call failed (%s)" % str(ex)
                logger.error("Query for year %s to Cisco EoX API failed (%s)" % (year, msg), exc_info=True)
                failed_years += [year]

                NotificationMessage.objects.create(
                    title="Initial data import failed",
                    summary_message="Unable to collect Cisco EoX data for year %d" % year,
                    detailed_message=msg,
                    type=NotificationMessage.MESSAGE_ERROR
                )

            except Exception as ex:
                msg = "Unexpected Exception, cannot access the Cisco API. Please ensure that the server is " \
                      "connected to the internet and that the authentication settings are valid."
                logger.error("Query for year %s to Cisco EoX API failed (%s)" % (year, msg), exc_info=True)
                failed_years += [year]

                NotificationMessage.objects.create(
                    title="Initial data import failed",
                    summary_message="Unable to collect Cisco EoX data for year %d" % year,
                    detailed_message=msg,
                    type=NotificationMessage.MESSAGE_ERROR
                )

        # update local database (asynchronous task)
        if len(all_records) != 0:
            tasks = [
                update_local_database_records.s({}, all_records[0]["year"], all_records[0]["records"])
            ]
            for r in all_records[1:]:
                tasks.append(update_local_database_records.s(r["year"], r["records"]))

            tasks.append(notify_initial_import_result.s())
            chain(*tasks).apply_async()

    time.sleep(10)
    # remove in progress flag with the cache
    cache.delete("CISCO_EOX_INITIAL_SYN_IN_PROGRESS")

    success_msg = ",".join([str(e) for e in successful_years])
    if len(success_msg) == 0:
        success_msg = "None"
    failed_msg = ""
    if len(failed_years) != 0:
        failed_msg = " (for %s the synchronization failed)" % ",".join([str(e) for e in failed_years])

    return {
        "status_message": "The EoX data were successfully downloaded for the following years: %s%s" % (success_msg,
                                                                                                       failed_msg)
    }
Exemplo n.º 11
0
def execute_task_to_synchronize_cisco_eox_states(self, ignore_periodic_sync_flag=False):
    """
    This task synchronize the local database with the Cisco EoX API. It executes all configured queries and stores the
    results in the local database. There are two types of operation:
      * cisco_eox_api_auto_sync_auto_create_elements is set to true - will create any element which is not part of the
                                                                      blacklist and not in the database
      * cisco_eox_api_auto_sync_auto_create_elements is set to false - will only update entries, which are already
                                                                       included in the database
    :return:
    """
    app_config = AppSettings()
    run_task = app_config.is_periodic_sync_enabled()

    if not (run_task or ignore_periodic_sync_flag):
        result = {
            "status_message": "task not enabled"
        }

    else:
        logger.info("start sync with Cisco EoX API...")
        self.update_state(state=TaskState.PROCESSING, meta={
            "status_message": "sync with Cisco EoX API..."
        })

        # read configuration for the Cisco EoX API synchronization
        queries = app_config.get_cisco_eox_api_queries_as_list()

        if len(queries) == 0:
            result = {
                "status_message": "No Cisco EoX API queries configured."
            }

            NotificationMessage.objects.create(
                title=NOTIFICATION_MESSAGE_TITLE,
                type=NotificationMessage.MESSAGE_WARNING,
                summary_message="There are no Cisco EoX API queries configured. Nothing to do.",
                detailed_message="There are no Cisco EoX API queries configured. Please configure at least on EoX API "
                                 "query in the settings or disable the periodic synchronization."
            )

        # update the local database with the Cisco EoX API
        else:
            # test Cisco EoX API access
            test_result = utils.check_cisco_eox_api_access(
                app_config.get_cisco_api_client_id(),
                app_config.get_cisco_api_client_secret(),
                False
            )

            if not test_result:
                msg = "Cannot contact Cisco EoX API, please verify your internet connection and access " \
                      "credentials."
                if not ignore_periodic_sync_flag:
                    NotificationMessage.objects.create(
                        title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR,
                        summary_message="The synchronization with the Cisco EoX API was not successful.",
                        detailed_message=msg
                    )

                result = {
                    "error_message": msg
                }

            else:
                # execute all queries from the configuration
                query_eox_records = {}
                failed_queries = []
                failed_query_msgs = {}
                successful_queries = []
                counter = 1

                for query in queries:
                    self.update_state(state=TaskState.PROCESSING, meta={
                        "status_message": "send query <code>%s</code> to the Cisco EoX API (<strong>%d of "
                                          "%d</strong>)..." % (query, counter, len(queries))
                    })

                    # wait some time between the query calls
                    time.sleep(int(app_config.get_cisco_eox_api_sync_wait_time()))
                    try:
                        query_eox_records[query] = cisco_eox_api_crawler.get_raw_api_data(api_query=query)
                        successful_queries.append(query)

                    except CiscoApiCallFailed as ex:
                        msg = "Cisco EoX API call failed (%s)" % str(ex)
                        logger.error("Query %s to Cisco EoX API failed (%s)" % (query, msg), exc_info=True)
                        failed_queries.append(query)
                        failed_query_msgs[query] = str(ex)

                    except Exception as ex:
                        msg = "Unexpected Exception, cannot access the Cisco API. Please ensure that the server is " \
                              "connected to the internet and that the authentication settings are " \
                              "valid."
                        logger.error("Query %s to Cisco EoX API failed (%s)" % (query, msg), exc_info=True)
                        failed_queries.append(query)
                        failed_query_msgs[query] = str(ex)

                    counter += 1

                for key in query_eox_records:
                    amount_of_records = len(query_eox_records[key])
                    self.update_state(state=TaskState.PROCESSING, meta={
                        "status_message": "update database (query <code>%s</code>, processed <b>0</b> of "
                                          "<b>%d</b> results)..." % (key, amount_of_records)
                    })

                    # update database in a separate task
                    update_cisco_eox_records.apply_async(kwargs={
                        "records": query_eox_records[key]
                    })

                # view the queries in the detailed message and all messages (if there are some)
                detailed_message = "The following queries were executed:<br><ul style=\"text-align: left;\">"
                for fq in failed_queries:
                    detailed_message += "<li class=\"text-danger\"><code>%s</code> " \
                                        "(failed, %s)</li>" % (fq, failed_query_msgs.get(fq, "unknown"))
                for sq in successful_queries:
                    detailed_message += "<li><code>%s</code> (<b>affects %d products</b>, " \
                                        "success)</li>" % (sq, len(query_eox_records[sq]))
                detailed_message += "</ul>"

                # show the executed queries in the summary message
                if len(failed_queries) == 0 and len(successful_queries) != 0:
                    summary_html = "The following queries were successful executed: %s" % ", ".join(
                        ["<code>%s</code>" % query for query in successful_queries]
                    )
                    NotificationMessage.objects.create(
                        title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_SUCCESS,
                        summary_message="The synchronization with the Cisco EoX API was successful. " + summary_html,
                        detailed_message=detailed_message
                    )

                elif len(failed_queries) != 0 and len(successful_queries) == 0:
                    summary_html = "The following queries failed to execute: %s" % ", ".join(
                        ["<code>%s</code>" % query for query in failed_queries]
                    )
                    NotificationMessage.objects.create(
                        title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_ERROR,
                        summary_message="The synchronization with the Cisco EoX API was not successful. " + summary_html,
                        detailed_message=detailed_message
                    )

                else:
                    summary_html = "The following queries were successful executed: %s\n<br>The following queries " \
                                   "failed to execute: %s" % (
                                       ", ".join(["<code>%s</code>" % query for query in successful_queries]),
                                       ", ".join(["<code>%s</code>" % query for query in failed_queries])
                                   )
                    NotificationMessage.objects.create(
                        title=NOTIFICATION_MESSAGE_TITLE, type=NotificationMessage.MESSAGE_WARNING,
                        summary_message="The synchronization with the Cisco EoX API was partially "
                                        "successful. " + summary_html,
                        detailed_message=detailed_message
                    )

                result = {"status_message": "<p style=\"text-align: left;\">" + detailed_message + "</p>"}

                # if the task was executed eager, set state to SUCCESS (required for testing)
                if self.request.is_eager:
                    self.update_state(state=TaskState.SUCCESS, meta={"status_message": summary_html})

    # remove in progress flag with the cache
    cache.delete("CISCO_EOX_API_SYN_IN_PROGRESS")

    return result
Exemplo n.º 12
0
def status(request):
    """
    Status page for the Product Database
    """
    app_config = AppSettings()

    is_cisco_api_enabled = app_config.is_cisco_api_enabled()
    context = {
        "is_cisco_api_enabled": is_cisco_api_enabled
    }

    if is_cisco_api_enabled:
        # test access (once every 30 minutes)
        cisco_eox_api_test_successful = cache.get("CISCO_EOX_API_TEST", False)

        # defaults, overwritten if an exception is thrown
        cisco_eox_api_available = True
        cisco_eox_api_message = "successful connected to the Cisco EoX API"

        if not cisco_eox_api_test_successful:
            try:
                result = utils.check_cisco_eox_api_access(client_id=app_config.get_cisco_api_client_id(),
                                                          client_secret=app_config.get_cisco_api_client_secret(),
                                                          drop_credentials=False)
                cache.set("CISCO_EOX_API_TEST", result, 60 * 30)

            except Exception as ex:
                cisco_eox_api_available = True
                cisco_eox_api_message = str(ex)

        context["cisco_eox_api_available"] = cisco_eox_api_available
        context["cisco_eox_api_message"] = cisco_eox_api_message

    # determine worker status
    ws = WorkerState.objects.all()
    if ws.count() == 0:
        worker_status = """
            <div class="alert alert-danger" role="alert">
                <span class="fa fa-exclamation-circle"></span>
                <span class="sr-only">Error:</span>
                All backend worker offline, asynchronous and scheduled tasks are not executed.
            </div>"""
    else:
        alive_worker = False
        for w in ws:
            if w.is_alive():
                alive_worker = True
                break
        if alive_worker:
            worker_status = """
                <div class="alert alert-success" role="alert">
                    <span class="fa fa-info-circle"></span>
                    <span class="sr-only">Error:</span>
                    Backend worker found.
                </div>"""

        else:
            worker_status = """
                <div class="alert alert-warning" role="alert">
                    <span class="fa fa-exclamation-circle"></span>
                    <span class="sr-only">Error:</span>
                    Only unregistered backend worker found, asynchronous and scheduled tasks are not executed.
                    Please verify the state in the <a href="/productdb/admin">Django Admin</a> page.
                </div>"""

    context['worker_status'] = mark_safe(worker_status)

    return render(request, "config/status.html", context=context)
Exemplo n.º 13
0
def change_configuration(request):
    """
    change configuration of the Product Database
    """
    # read settings from configuration file
    app_config = AppSettings()

    # read settings from database
    hp_content_after, _ = TextBlock.objects.get_or_create(
        name=TextBlock.TB_HOMEPAGE_TEXT_AFTER_FAVORITE_ACTIONS
    )
    hp_content_before, _ = TextBlock.objects.get_or_create(
        name=TextBlock.TB_HOMEPAGE_TEXT_BEFORE_FAVORITE_ACTIONS
    )

    if request.method == "POST":
        # create a form instance and populate it with data from the request:
        form = SettingsForm(request.POST)
        if form.is_valid():
            # set common settings
            app_config.set_login_only_mode(form.cleaned_data["login_only_mode"])

            hp_content_before.html_content = form.cleaned_data["homepage_text_before"]
            hp_content_before.save()
            hp_content_after.html_content = form.cleaned_data["homepage_text_after"]
            hp_content_after.save()

            # set the Cisco API configuration options
            api_enabled = form.cleaned_data["cisco_api_enabled"]

            if not api_enabled:
                # api is disabled, reset values to default
                app_config.set_cisco_api_enabled(api_enabled)
                app_config.set_cisco_api_client_id("PlsChgMe")
                app_config.set_cisco_api_client_secret("PlsChgMe")
                app_config.set_periodic_sync_enabled(False)
                app_config.set_auto_create_new_products(False)
                app_config.set_cisco_eox_api_queries("")
                app_config.set_product_blacklist_regex("")
                app_config.set_cisco_eox_api_sync_wait_time("5")

            else:
                app_config.set_cisco_api_enabled(api_enabled)

                client_id = form.cleaned_data["cisco_api_client_id"] \
                    if form.cleaned_data["cisco_api_client_id"] != "" else "PlsChgMe"
                app_config.set_cisco_api_client_id(client_id)
                client_secret = form.cleaned_data["cisco_api_client_secret"] \
                    if form.cleaned_data["cisco_api_client_secret"] != "" else "PlsChgMe"
                app_config.set_cisco_api_client_secret(client_secret)

                if client_id != "PlsChgMe":
                    result = utils.check_cisco_eox_api_access(
                        form.cleaned_data["cisco_api_client_id"],
                        form.cleaned_data["cisco_api_client_secret"]
                    )

                    if result:
                        messages.success(request, "Successfully connected to the Cisco EoX API")

                    else:
                        messages.error(request, "Cannot contact the Cisco EoX API. Please contact your "
                                                "Administrator")

                else:
                    messages.info(
                        request,
                        "Please configure your Cisco API credentials within the Cisco API settings tab."
                    )

                app_config.set_periodic_sync_enabled(form.cleaned_data["eox_api_auto_sync_enabled"])
                app_config.set_internal_product_id_label(form.cleaned_data["internal_product_id_label"])
                app_config.set_auto_create_new_products(form.cleaned_data["eox_auto_sync_auto_create_elements"])
                app_config.set_cisco_eox_api_queries(form.cleaned_data["eox_api_queries"])
                app_config.set_product_blacklist_regex(form.cleaned_data["eox_api_blacklist"])
                if form.cleaned_data["eox_api_wait_time"]:
                    app_config.set_cisco_eox_api_sync_wait_time(form.cleaned_data["eox_api_wait_time"])

            # expire cached settings
            cache.delete("LOGIN_ONLY_MODE_SETTING")

            messages.success(request, "Settings saved successfully")
            return redirect(resolve_url("productdb_config:change_settings"))

        else:
            messages.error(request, "Invalid configuration option detected, please check it below.")

    else:
        form = SettingsForm()
        form.fields['cisco_api_enabled'].initial = app_config.is_cisco_api_enabled()
        form.fields['login_only_mode'].initial = app_config.is_login_only_mode()
        form.fields['internal_product_id_label'].initial = app_config.get_internal_product_id_label()
        form.fields['cisco_api_client_id'].initial = app_config.get_cisco_api_client_id()
        form.fields['cisco_api_client_secret'].initial = app_config.get_cisco_api_client_secret()
        form.fields['eox_api_auto_sync_enabled'].initial = app_config.is_periodic_sync_enabled()
        form.fields['eox_auto_sync_auto_create_elements'].initial = app_config.is_auto_create_new_products()
        form.fields['eox_api_queries'].initial = app_config.get_cisco_eox_api_queries()
        form.fields['eox_api_blacklist'].initial = app_config.get_product_blacklist_regex()
        form.fields['eox_api_wait_time'].initial = app_config.get_cisco_eox_api_sync_wait_time()
        form.fields['homepage_text_before'].initial = hp_content_before.html_content
        form.fields['homepage_text_after'].initial = hp_content_after.html_content

    context = {
        "form": form,
        "is_cisco_api_enabled": app_config.is_cisco_api_enabled(),
        "is_cisco_eox_api_auto_sync_enabled": app_config.is_periodic_sync_enabled()
    }
    return render(request, "config/change_configuration.html", context=context)