def send_email(to, subject, template): msg = Message( subject, recipients=[to], html=template, sender=current_app.config["MAIL_DEFAULT_SENDER"], ) try: mail.send(msg) except ConnectionError as e: log(log.ERROR, "Connection error. Turn on your router: %s", e)
def delete_user(self, name): user_dn = self.FORMAT_USER_DN.format(name) with ldap3.Connection( self.server, user=config.LDAP_USER, password=config.LDAP_PASS ) as connection: success = connection.delete(user_dn) if not success: log(log.ERROR, "Cannot delete user [%s]", user_dn) log(log.ERROR, "%s", connection.result) return False return True
def generator(file_pkl, file_data): """Simple program that greets NAME for a total of COUNT times.""" # LOAD PKL FILE xgb_model = None try: xgb_model = pickle.load(open(file_pkl, 'rb')) except Exception: sys.stderr.write( "The model file uploaded cannot be parsed." " Please check you have uploaded the correct file." ) return # Separate test (evaluation) dataset that doesn't include the output test_data = None try: test_data = pd.read_csv(file_data) except Exception: sys.stderr.write( "The Testing Dataset file uploaded cannot be parsed." " Please check you have uploaded the correct file." ) return # Choose the same columns you trained the model with start_time = datetime.now() X_new = test_data[test_data.columns] ypred = xgb_model.predict(xgb.DMatrix(X_new)) test_data["predictions"] = ypred file_uuid = str(uuid.uuid4()) background_file = os.path.join(PATH_TO_RESULT, f"background_{file_uuid}.csv") test_data.to_csv(background_file) log(log.DEBUG, "Prediction. Took time: [%s]", datetime.now() - start_time) explainerXGB = shap.TreeExplainer(xgb_model) shap_values_XGB_test = explainerXGB.shap_values(X_new) df_shap_XGB_test = pd.DataFrame(shap_values_XGB_test, columns=X_new.columns.values) explainer_file = os.path.join(PATH_TO_RESULT, f"explainer_{file_uuid}.csv") df_shap_XGB_test.to_csv(explainer_file) # PLOT # load JS visualization code to notebook shap.initjs() # summarize the effects of all the features shap.summary_plot(shap_values_XGB_test, X_new, show=False) plot_file = os.path.join(PATH_TO_RESULT, f"plot_{file_uuid}.png") plt.savefig(plot_file, bbox_inches='tight') print(background_file) print(explainer_file) print(plot_file)
def __init__(self): if conf.REMOTE_SHELL_SERVER: super().__init__( conf.REMOTE_SHELL_SERVER, port=conf.REMOTE_SHELL_PORT, user=conf.REMOTE_SHELL_USER, passwd=conf.REMOTE_SHELL_PASS, ) else: log(log.WARNING, "RemoteShell: please define REMOTE_SHELL_SERVER") self.client = None
def get_invoice(self, order_id): response = retry_get_request( f"{conf.VIDAXL_API_BASE_URL}/api_customer/orders/{order_id}/documents", auth=self.basic_auth, ) if response.status_code == 200: log(log.INFO, "%s", response.text) return json.loads(response.text) else: log(log.ERROR, "Invalid response, status code: [%s]", response.status_code) return None
def reset_config_parameters(shop_id): """[Delete configurations from DB by shop] Args: shop_id ([int]): [Shop ID] """ configs = Configuration.query.filter( Configuration.shop_id == shop_id).filter( or_(Configuration.name == v for v in PARAMETERS)).all() for config in configs: config.delete() log(log.INFO, "Configs was deleted in Shop ID[%d]", shop_id)
def login(): form = LoginForm(request.form) log(log.INFO, '/login') if form.validate_on_submit(): user = User.authenticate(form.user_name.data, form.password.data) if user is not None: login_user(user) flash("Login successful.", "success") return redirect(url_for("main.index")) flash("Wrong user name or password.", "danger") log(log.WARNING, "Invalid user data") return render_template("login.html", form=form)
def delete_product(self, prod_id: int, product_key: str): """deletes product by id (archive product) Arguments: prod_id {int} -- Invoice Ninja Product ID """ log(log.DEBUG, "NinjaApi.delete_product %d", prod_id) return self.do_put( "{}products/{}?action=delete".format(self.BASE_URL, prod_id), id=prod_id, product_key=product_key, )
def delete(): log(log.INFO, '%s /account_ext_delete', request.method) if not hasValidIdentificator(request): flash(UNKNOWN_ID, 'danger') return redirect(url_for('main.accounts')) extension = AccountExtension.query.filter(AccountExtension.id == int(request.args['id'])).first() if not extension: flash(UNKNOWN_ID, 'danger') return redirect(url_for('main.accounts')) account_id = extension.account_id extension.delete() return redirect(url_for('account.edit', id=account_id))
def do_delete(self, url: str): headers = {"X-Ninja-Token": self.NINJA_TOKEN} try: response = requests.delete(url, headers=headers) except requests.exceptions.ConnectionError: log(log.ERROR, "NinjaApi wrong NINJA_API_BASE_URL") return None try: response.raise_for_status() except requests.HTTPError as error: log(log.ERROR, "NinjaApi.HTTPError: %s", error) return response.ok
def get_file(self) -> bool: file_url = self.scrapper(self.week_no, self.year_no) if not file_url: log(log.ERROR, "File URL is not found.") return False file = requests.get(file_url, stream=True) file.raise_for_status() self.file = tempfile.NamedTemporaryFile(mode="wb+") for chunk in file.iter_content(chunk_size=4096): self.file.write(chunk) self.file.seek(0) return True
def get_product(self, product_id: int): resp = requests.get( self.base_url + f"/admin/api/{self.version_api}/products/{product_id}.json", headers=self.headers, ) if resp.status_code == 200: return resp.json() else: log(log.ERROR, "Response is invalid. Status code: [%s]", resp.status_code) raise Exception
def update_categories(shop_id: int, file): lines = file.readlines() if lines: Category.query.filter(Category.shop_id == shop_id).delete() for line in lines: path = line.decode().strip(" \t\n\r") if path: log(log.INFO, "Add category: [%s]", path) Category( shop_id=shop_id, path=path, ).save()
def do_post(self, url: str, **data): headers = {"X-Ninja-Token": self.NINJA_TOKEN} try: response = requests.post(url, headers=headers, data=data) except requests.exceptions.ConnectionError: log(log.ERROR, "NinjaApi wrong NINJA_API_BASE_URL") return None try: response.raise_for_status() except requests.HTTPError as error: log(log.ERROR, "NinjaApi.HTTPError: %s", error) return response.json() if response.ok else None
def add_client(self, name: str): """adds new client curl -X POST ninja.test/api/v1/clients -H "Content-Type:application/json" \ -d '{"name":"Client"}' -H "X-Ninja-Token: TOKEN" Arguments: name {str} -- Ninja Client Name """ log(log.DEBUG, "NinjaApi.add_client %s", name) res = self.do_post(self.BASE_URL + "clients", name=name) if not res or not res["data"]: return res return NinjaClient(res["data"])
def update(self) -> None: log(log.INFO, "Update file from [%s]", conf.STATISTICS_DATABASE_URL) data = [] for file_name in DATASET_MAP_JSON.values(): url = urllib.parse.urljoin(conf.STATISTICS_DATABASE_URL, file_name) log(log.INFO, "Download [%s] from [%s]", file_name, url) res = requests.get(url) # data = DATASET_MAP_JSON.values() data.append(url) with open(pathlib.Path(EXCEL_FILES_DIR) / file_name, "wb") as f: f.write(res.content) return data
def create(cls, title: str, shop_id: int): resp = requests.post( cls.BASE_URL + f"/admin/api/{cls.VERSION_API}/custom_collections.json", headers=cls.headers(cls.shop_id), json={"custom_collection": { "title": title }}) if resp.status_code != 201: log(log.ERROR, "Invalid response, status code: [%s]", resp.status_code) return None return cls(shop_id, resp.json())
def chart_data(self, chart_name: str, options: dict = {}) -> dict: if chart_name == ANALYTICS: return self.analytics_data(options) if chart_name not in DATASET_MAP_JSON: log(log.WARNING, "Asked unknown chart_name: [%s]", chart_name) return {} bank_view = options["bankView"] if "bankView" in options else False file_name = DATASET_MAP_JSON[chart_name] data = self.get_csv_file_data(file_name) if chart_name in DATA_PROCESSOR: return DATA_PROCESSOR[chart_name][bank_view](data, options) return data
def resellers(): log(log.INFO, '/resellers') if current_user.user_type.name not in ['super_admin', 'admin']: return redirect(url_for("main.index")) return render_template( 'index.html', main_content='Resellers', table_data=[ i.to_dict() for i in Reseller.query.filter(Reseller.deleted == False) ], # noqa E712 columns=Reseller.columns(), edit_href=url_for('reseller.edit'))
def add(): log(log.INFO, '%s /account_extension_add', request.method) log(log.DEBUG, 'args: %s', request.args) if not hasValidIdentificator(request): flash(UNKNOWN_ID, 'danger') return redirect(url_for('main.accounts')) account_id = int(request.args['id']) form = AccountExtensionForm(id=account_id) form.products = Product.query.filter(Product.deleted == False).order_by(Product.name).all() # noqa E712 form.resellers = organize_list_starting_with_value(Reseller.query.order_by(Reseller.name).all(), 'NITRIX') # noqa E712 form.is_edit = False form.save_route = url_for('account_extension.save_new') form.close_button = url_for('account.edit', id=account_id) return render_template("account_extension.html", form=form)
def get_typed_value(value: str, value_type: str): switch = { "bool": lambda x: value in ("True", "true", "Y", "y"), "float": lambda x: float(x), "int": lambda x: int(x), "str": lambda x: str(x), "datetime": lambda x: datetime.fromisoformat(x), } if value_type not in switch: log(log.ERROR, "No value type found: [%s]", value_type) return value return switch[value_type](value)
def scrapper(self, week: int, year: int) -> str or None: links = self.links if not links: options = webdriver.ChromeOptions() options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--headless") browser = webdriver.Chrome(options=options, executable_path=conf.CHROME_DRIVER_PATH) log(log.INFO, "Start get url CSX") browser.get(self.URL) log(log.INFO, "Got url CSX") generated_html = browser.page_source soup = BeautifulSoup(generated_html, "html.parser") links = soup.find_all("a", class_="module_link") while len(links) < 53: browser.get(self.URL) generated_html = browser.page_source soup = BeautifulSoup(generated_html, "html.parser") links = soup.find_all("a", class_="module_link") time.sleep(1) self.links = links for i in links: scrap_data = i.span.text.split() scrap_year = scrap_data[0] scrap_week = scrap_data[2] if scrap_year == str(year) and scrap_week == str(week): link = i["href"] log(log.INFO, "Found pdf link: [%s]", link) return link log(log.WARNING, "Links not found") return None
def scrapper(self, week: int, year: int) -> str or None: if conf.CURRENT_YEAR != year: log(log.WARNING, "Links not found") return None links = self.links options = webdriver.ChromeOptions() options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--headless") browser = webdriver.Chrome( options=options, executable_path=conf.CHROME_DRIVER_PATH ) log(log.INFO, "Start get url Union") browser.get(self.URL) log(log.INFO, "Got url Union") generated_html = browser.page_source soup = BeautifulSoup(generated_html, "html.parser") links = soup.find_all("a", class_="pdf") for i in links: scrap_data = i.text.split() scrap_week = scrap_data[1] if str(week) == scrap_week: link = "https://www.up.com" + i["href"] log(log.INFO, "Found pdf link: [%s]", link) return link
def scrapper(self, week: int, year: int) -> str or None: links = self.links options = webdriver.ChromeOptions() options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--headless") browser = webdriver.Chrome(options=options, executable_path=conf.CHROME_DRIVER_PATH) log(log.INFO, "Start get url Kansas City") browser.get(self.URL) log(log.INFO, "Got url Kansas City") generated_html = browser.page_source soup = BeautifulSoup(generated_html, "html.parser") links = soup.find_all("a", class_="ext-link") for i in links: if len(str(week)) == 1: week = f"0{week}" scrap_data = i.attrs["href"].split("/")[6] scrap_date = scrap_data.split("-") scrap_week = scrap_date[1] scrap_year = scrap_date[4] if str(week) == scrap_week and str(year) == scrap_year: link = "https://investors.kcsouthern.com" + i.attrs["href"] log(log.INFO, "Found pdf link: [%s]", link) return link log(log.WARNING, "Links not found") return None
def handleLogin(): if request.method == "POST": username = request.form["username"].lower() password = request.form["password"] user = dbClasses.User.query.filter_by(username=username).first() if user is not None and username == user.username and user.check_password(password): logger.log(25, configuration.LogFormat() + username + " logged in") login_user(user) return redirect("/") else: logger.log(25, configuration.LogFormat() + "Attempted log in with username: "******"login.html", className = "warning", message="Wrong username or password") else: return render_template("login.html")
def decorated_function(*args, **kwargs): if not shopify.Session.secret: shopify.Session.setup( api_key=current_app.config["SHOPIFY_API_KEY"], secret=current_app.config["SHOPIFY_SECRET"], ) if not shopify.Session.validate_params(request.args): log(log.ERROR, "shopify_auth: Wrong arguments in the request") return redirect(url_for("shopify.install", **request.args)) shop = Shop.query.filter_by( name=request.args.get("shop", "")).first() if not shop: log(log.ERROR, "shopify_auth: Unknown shop: [%s]", shop) return redirect(url_for("shopify.install", **request.args)) session["shopify_token"] = shop.access_token session["shop"] = shop.name session["shopify_id"] = shop.id else: if shopify.Session.secret != current_app.config["SHOPIFY_SECRET"]: log(log.ERROR, "shopify_auth: Wrong arguments in the request") return redirect(url_for("shopify.install", **request.args)) name = request.args.get("shop", "") shop = Shop.query.filter_by(name=name).first() if not shop: log(log.ERROR, "Unknown shop: [%s]", name) return redirect(url_for("shopify.install", **request.args)) return f(*args, **kwargs)
def get_specific_custom_collections(self, *collection_ids: int) -> list: collection_ids = ','.join([str(arg) for arg in collection_ids]) resp = requests.get( self.BASE_URL + f"/admin/api/{self.VERSION_API}/custom_collections.json?ids={collection_ids}", headers=self.headers(self.shop_id)) if resp.status_code == 200: custom_collections = resp.json().get("custom_collections", "") if custom_collections: return custom_collections else: log(log.DEBUG, "No specific_collections") else: log(log.ERROR, "Invalid response, status code: [%s]", resp.status_code)
def save(): log(log.INFO, '/reseller_product_save') form = ResellerProductForm(request.form) if not form.validate_on_submit(): flash('Form validation error', 'danger') log(log.WARNING, "Form validation error %s", form.errors) return redirect(url_for('reseller.edit', id=form.reseller_id.data)) product = ResellerProduct.query.filter( ResellerProduct.id == form.id.data).first() if product: product.reseller_id = form.reseller_id.data product.product_id = form.product_id.data product.months = form.months.data product.init_price = form.init_price.data product.ext_price = form.ext_price.data else: product = ResellerProduct.query.filter( ResellerProduct.reseller_id == form.reseller_id.data).filter( ResellerProduct.product_id == form.product_id.data).filter( ResellerProduct.months == form.months.data).first() if product: flash('Reseller already have this product', 'danger') return redirect(url_for('reseller.edit', id=form.reseller_id.data)) product = ResellerProduct(reseller_id=form.reseller_id.data, product_id=form.product_id.data, months=form.months.data, init_price=form.init_price.data, ext_price=form.ext_price.data) product.save() if ninja.configured: product_key = ninja_product_name(product.product.name, product.months) if form.id.data < 0: ninja_product = ninja.add_product(product_key=product_key, notes=product.reseller.name, cost=product.init_price) if ninja_product: product.ninja_product_id = ninja_product.id product.save() else: ninja_product = ninja.get_product(product.ninja_product_id) if ninja_product: ninja.update_product(ninja_product.id, product_key=product_key, notes=product.reseller.name, cost=product.init_price) return redirect(url_for('reseller.edit', id=form.reseller_id.data))
def save_update(): log(log.INFO, '%s /account_ext_save_update', request.method) form = AccountExtensionForm(request.form) if not form.validate_on_submit(): flash(VALIDATION_ERROR, 'danger') log(log.WARNING, VALIDATION_ERROR, form.errors) return redirect(url_for('account.edit', id=form.id.data)) extension = AccountExtension.query.filter(AccountExtension.id == form.id.data).first() extension.reseller_id = form.reseller_id.data extension.product_id = form.product_id.data extension.months = form.months.data extension.extension_date = form.extension_date.data extension.end_date = extension.extension_date + relativedelta(months=extension.months) account_id = extension.account_id extension.save() return redirect(url_for('account.edit', id=account_id))
def run_scrap(): try: data_scrap() except Exception as err: # send email on error import traceback body = f"Error {str(err)}\n" body += traceback.format_exc() msg = Message(subject="Error", body=body, recipients=conf.MAIL_RECIPIENTS.split(";")) log(log.INFO, "Mail sending...") with open(f"{LOGGER_NAME}.log", "r") as att: msg.attach(att.name, "text/plain", att.read()) mail.send(msg) assert msg