def send_email(to, subject, template):
msg = Message(
subject,
recipients=[to],
html=template,
sender=current_app.config["MAIL_DEFAULT_SENDER"],
)
try:
mail.send(msg)
except ConnectionError as e:
log(log.ERROR, "Connection error. Turn on your router: %s", e)
def delete_user(self, name):
user_dn = self.FORMAT_USER_DN.format(name)
with ldap3.Connection(
self.server, user=config.LDAP_USER, password=config.LDAP_PASS
) as connection:
success = connection.delete(user_dn)
if not success:
log(log.ERROR, "Cannot delete user [%s]", user_dn)
log(log.ERROR, "%s", connection.result)
return False
return True
def generator(file_pkl, file_data):
"""Simple program that greets NAME for a total of COUNT times."""
# LOAD PKL FILE
xgb_model = None
try:
xgb_model = pickle.load(open(file_pkl, 'rb'))
except Exception:
sys.stderr.write(
"The model file uploaded cannot be parsed."
" Please check you have uploaded the correct file."
)
return
# Separate test (evaluation) dataset that doesn't include the output
test_data = None
try:
test_data = pd.read_csv(file_data)
except Exception:
sys.stderr.write(
"The Testing Dataset file uploaded cannot be parsed."
" Please check you have uploaded the correct file."
)
return
# Choose the same columns you trained the model with
start_time = datetime.now()
X_new = test_data[test_data.columns]
ypred = xgb_model.predict(xgb.DMatrix(X_new))
test_data["predictions"] = ypred
file_uuid = str(uuid.uuid4())
background_file = os.path.join(PATH_TO_RESULT, f"background_{file_uuid}.csv")
test_data.to_csv(background_file)
log(log.DEBUG, "Prediction. Took time: [%s]", datetime.now() - start_time)
explainerXGB = shap.TreeExplainer(xgb_model)
shap_values_XGB_test = explainerXGB.shap_values(X_new)
df_shap_XGB_test = pd.DataFrame(shap_values_XGB_test, columns=X_new.columns.values)
explainer_file = os.path.join(PATH_TO_RESULT, f"explainer_{file_uuid}.csv")
df_shap_XGB_test.to_csv(explainer_file)
# PLOT
# load JS visualization code to notebook
shap.initjs()
# summarize the effects of all the features
shap.summary_plot(shap_values_XGB_test, X_new, show=False)
plot_file = os.path.join(PATH_TO_RESULT, f"plot_{file_uuid}.png")
plt.savefig(plot_file, bbox_inches='tight')
print(background_file)
print(explainer_file)
print(plot_file)
def __init__(self):
if conf.REMOTE_SHELL_SERVER:
super().__init__(
conf.REMOTE_SHELL_SERVER,
port=conf.REMOTE_SHELL_PORT,
user=conf.REMOTE_SHELL_USER,
passwd=conf.REMOTE_SHELL_PASS,
)
else:
log(log.WARNING, "RemoteShell: please define REMOTE_SHELL_SERVER")
self.client = None
def get_invoice(self, order_id):
response = retry_get_request(
f"{conf.VIDAXL_API_BASE_URL}/api_customer/orders/{order_id}/documents",
auth=self.basic_auth,
)
if response.status_code == 200:
log(log.INFO, "%s", response.text)
return json.loads(response.text)
else:
log(log.ERROR, "Invalid response, status code: [%s]", response.status_code)
return None
def reset_config_parameters(shop_id):
"""[Delete configurations from DB by shop]
Args:
shop_id ([int]): [Shop ID]
"""
configs = Configuration.query.filter(
Configuration.shop_id == shop_id).filter(
or_(Configuration.name == v for v in PARAMETERS)).all()
for config in configs:
config.delete()
log(log.INFO, "Configs was deleted in Shop ID[%d]", shop_id)
def login():
form = LoginForm(request.form)
log(log.INFO, '/login')
if form.validate_on_submit():
user = User.authenticate(form.user_name.data, form.password.data)
if user is not None:
login_user(user)
flash("Login successful.", "success")
return redirect(url_for("main.index"))
flash("Wrong user name or password.", "danger")
log(log.WARNING, "Invalid user data")
return render_template("login.html", form=form)
def delete_product(self, prod_id: int, product_key: str):
"""deletes product by id (archive product)
Arguments:
prod_id {int} -- Invoice Ninja Product ID
"""
log(log.DEBUG, "NinjaApi.delete_product %d", prod_id)
return self.do_put(
"{}products/{}?action=delete".format(self.BASE_URL, prod_id),
id=prod_id,
product_key=product_key,
)
def delete():
log(log.INFO, '%s /account_ext_delete', request.method)
if not hasValidIdentificator(request):
flash(UNKNOWN_ID, 'danger')
return redirect(url_for('main.accounts'))
extension = AccountExtension.query.filter(AccountExtension.id == int(request.args['id'])).first()
if not extension:
flash(UNKNOWN_ID, 'danger')
return redirect(url_for('main.accounts'))
account_id = extension.account_id
extension.delete()
return redirect(url_for('account.edit', id=account_id))
def do_delete(self, url: str):
headers = {"X-Ninja-Token": self.NINJA_TOKEN}
try:
response = requests.delete(url, headers=headers)
except requests.exceptions.ConnectionError:
log(log.ERROR, "NinjaApi wrong NINJA_API_BASE_URL")
return None
try:
response.raise_for_status()
except requests.HTTPError as error:
log(log.ERROR, "NinjaApi.HTTPError: %s", error)
return response.ok
def get_file(self) -> bool:
file_url = self.scrapper(self.week_no, self.year_no)
if not file_url:
log(log.ERROR, "File URL is not found.")
return False
file = requests.get(file_url, stream=True)
file.raise_for_status()
self.file = tempfile.NamedTemporaryFile(mode="wb+")
for chunk in file.iter_content(chunk_size=4096):
self.file.write(chunk)
self.file.seek(0)
return True
def get_product(self, product_id: int):
resp = requests.get(
self.base_url +
f"/admin/api/{self.version_api}/products/{product_id}.json",
headers=self.headers,
)
if resp.status_code == 200:
return resp.json()
else:
log(log.ERROR, "Response is invalid. Status code: [%s]",
resp.status_code)
raise Exception
def update_categories(shop_id: int, file):
lines = file.readlines()
if lines:
Category.query.filter(Category.shop_id == shop_id).delete()
for line in lines:
path = line.decode().strip(" \t\n\r")
if path:
log(log.INFO, "Add category: [%s]", path)
Category(
shop_id=shop_id,
path=path,
).save()
def do_post(self, url: str, **data):
headers = {"X-Ninja-Token": self.NINJA_TOKEN}
try:
response = requests.post(url, headers=headers, data=data)
except requests.exceptions.ConnectionError:
log(log.ERROR, "NinjaApi wrong NINJA_API_BASE_URL")
return None
try:
response.raise_for_status()
except requests.HTTPError as error:
log(log.ERROR, "NinjaApi.HTTPError: %s", error)
return response.json() if response.ok else None
def add_client(self, name: str):
"""adds new client
curl -X POST ninja.test/api/v1/clients -H "Content-Type:application/json" \
-d '{"name":"Client"}' -H "X-Ninja-Token: TOKEN"
Arguments:
name {str} -- Ninja Client Name
"""
log(log.DEBUG, "NinjaApi.add_client %s", name)
res = self.do_post(self.BASE_URL + "clients", name=name)
if not res or not res["data"]:
return res
return NinjaClient(res["data"])
def update(self) -> None:
log(log.INFO, "Update file from [%s]", conf.STATISTICS_DATABASE_URL)
data = []
for file_name in DATASET_MAP_JSON.values():
url = urllib.parse.urljoin(conf.STATISTICS_DATABASE_URL, file_name)
log(log.INFO, "Download [%s] from [%s]", file_name, url)
res = requests.get(url)
# data = DATASET_MAP_JSON.values()
data.append(url)
with open(pathlib.Path(EXCEL_FILES_DIR) / file_name, "wb") as f:
f.write(res.content)
return data
def create(cls, title: str, shop_id: int):
resp = requests.post(
cls.BASE_URL +
f"/admin/api/{cls.VERSION_API}/custom_collections.json",
headers=cls.headers(cls.shop_id),
json={"custom_collection": {
"title": title
}})
if resp.status_code != 201:
log(log.ERROR, "Invalid response, status code: [%s]",
resp.status_code)
return None
return cls(shop_id, resp.json())
def chart_data(self, chart_name: str, options: dict = {}) -> dict:
if chart_name == ANALYTICS:
return self.analytics_data(options)
if chart_name not in DATASET_MAP_JSON:
log(log.WARNING, "Asked unknown chart_name: [%s]", chart_name)
return {}
bank_view = options["bankView"] if "bankView" in options else False
file_name = DATASET_MAP_JSON[chart_name]
data = self.get_csv_file_data(file_name)
if chart_name in DATA_PROCESSOR:
return DATA_PROCESSOR[chart_name][bank_view](data, options)
return data
def resellers():
log(log.INFO, '/resellers')
if current_user.user_type.name not in ['super_admin', 'admin']:
return redirect(url_for("main.index"))
return render_template(
'index.html',
main_content='Resellers',
table_data=[
i.to_dict()
for i in Reseller.query.filter(Reseller.deleted == False)
], # noqa E712
columns=Reseller.columns(),
edit_href=url_for('reseller.edit'))
def add():
log(log.INFO, '%s /account_extension_add', request.method)
log(log.DEBUG, 'args: %s', request.args)
if not hasValidIdentificator(request):
flash(UNKNOWN_ID, 'danger')
return redirect(url_for('main.accounts'))
account_id = int(request.args['id'])
form = AccountExtensionForm(id=account_id)
form.products = Product.query.filter(Product.deleted == False).order_by(Product.name).all() # noqa E712
form.resellers = organize_list_starting_with_value(Reseller.query.order_by(Reseller.name).all(), 'NITRIX') # noqa E712
form.is_edit = False
form.save_route = url_for('account_extension.save_new')
form.close_button = url_for('account.edit', id=account_id)
return render_template("account_extension.html", form=form)
def get_typed_value(value: str, value_type: str):
switch = {
"bool": lambda x: value in ("True", "true", "Y", "y"),
"float": lambda x: float(x),
"int": lambda x: int(x),
"str": lambda x: str(x),
"datetime": lambda x: datetime.fromisoformat(x),
}
if value_type not in switch:
log(log.ERROR, "No value type found: [%s]", value_type)
return value
return switch[value_type](value)
def scrapper(self, week: int, year: int) -> str or None:
links = self.links
if not links:
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--headless")
browser = webdriver.Chrome(options=options,
executable_path=conf.CHROME_DRIVER_PATH)
log(log.INFO, "Start get url CSX")
browser.get(self.URL)
log(log.INFO, "Got url CSX")
generated_html = browser.page_source
soup = BeautifulSoup(generated_html, "html.parser")
links = soup.find_all("a", class_="module_link")
while len(links) < 53:
browser.get(self.URL)
generated_html = browser.page_source
soup = BeautifulSoup(generated_html, "html.parser")
links = soup.find_all("a", class_="module_link")
time.sleep(1)
self.links = links
for i in links:
scrap_data = i.span.text.split()
scrap_year = scrap_data[0]
scrap_week = scrap_data[2]
if scrap_year == str(year) and scrap_week == str(week):
link = i["href"]
log(log.INFO, "Found pdf link: [%s]", link)
return link
log(log.WARNING, "Links not found")
return None
def scrapper(self, week: int, year: int) -> str or None:
if conf.CURRENT_YEAR != year:
log(log.WARNING, "Links not found")
return None
links = self.links
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--headless")
browser = webdriver.Chrome(
options=options, executable_path=conf.CHROME_DRIVER_PATH
)
log(log.INFO, "Start get url Union")
browser.get(self.URL)
log(log.INFO, "Got url Union")
generated_html = browser.page_source
soup = BeautifulSoup(generated_html, "html.parser")
links = soup.find_all("a", class_="pdf")
for i in links:
scrap_data = i.text.split()
scrap_week = scrap_data[1]
if str(week) == scrap_week:
link = "https://www.up.com" + i["href"]
log(log.INFO, "Found pdf link: [%s]", link)
return link
def scrapper(self, week: int, year: int) -> str or None:
links = self.links
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--headless")
browser = webdriver.Chrome(options=options,
executable_path=conf.CHROME_DRIVER_PATH)
log(log.INFO, "Start get url Kansas City")
browser.get(self.URL)
log(log.INFO, "Got url Kansas City")
generated_html = browser.page_source
soup = BeautifulSoup(generated_html, "html.parser")
links = soup.find_all("a", class_="ext-link")
for i in links:
if len(str(week)) == 1:
week = f"0{week}"
scrap_data = i.attrs["href"].split("/")[6]
scrap_date = scrap_data.split("-")
scrap_week = scrap_date[1]
scrap_year = scrap_date[4]
if str(week) == scrap_week and str(year) == scrap_year:
link = "https://investors.kcsouthern.com" + i.attrs["href"]
log(log.INFO, "Found pdf link: [%s]", link)
return link
log(log.WARNING, "Links not found")
return None
def handleLogin():
if request.method == "POST":
username = request.form["username"].lower()
password = request.form["password"]
user = dbClasses.User.query.filter_by(username=username).first()
if user is not None and username == user.username and user.check_password(password):
logger.log(25, configuration.LogFormat() + username + " logged in")
login_user(user)
return redirect("/")
else:
logger.log(25, configuration.LogFormat() + "Attempted log in with username: "******"login.html", className = "warning", message="Wrong username or password")
else:
return render_template("login.html")
def decorated_function(*args, **kwargs):
if not shopify.Session.secret:
shopify.Session.setup(
api_key=current_app.config["SHOPIFY_API_KEY"],
secret=current_app.config["SHOPIFY_SECRET"],
)
if not shopify.Session.validate_params(request.args):
log(log.ERROR, "shopify_auth: Wrong arguments in the request")
return redirect(url_for("shopify.install", **request.args))
shop = Shop.query.filter_by(
name=request.args.get("shop", "")).first()
if not shop:
log(log.ERROR, "shopify_auth: Unknown shop: [%s]", shop)
return redirect(url_for("shopify.install", **request.args))
session["shopify_token"] = shop.access_token
session["shop"] = shop.name
session["shopify_id"] = shop.id
else:
if shopify.Session.secret != current_app.config["SHOPIFY_SECRET"]:
log(log.ERROR, "shopify_auth: Wrong arguments in the request")
return redirect(url_for("shopify.install", **request.args))
name = request.args.get("shop", "")
shop = Shop.query.filter_by(name=name).first()
if not shop:
log(log.ERROR, "Unknown shop: [%s]", name)
return redirect(url_for("shopify.install", **request.args))
return f(*args, **kwargs)
def get_specific_custom_collections(self, *collection_ids: int) -> list:
collection_ids = ','.join([str(arg) for arg in collection_ids])
resp = requests.get(
self.BASE_URL +
f"/admin/api/{self.VERSION_API}/custom_collections.json?ids={collection_ids}",
headers=self.headers(self.shop_id))
if resp.status_code == 200:
custom_collections = resp.json().get("custom_collections", "")
if custom_collections:
return custom_collections
else:
log(log.DEBUG, "No specific_collections")
else:
log(log.ERROR, "Invalid response, status code: [%s]",
resp.status_code)
def save():
log(log.INFO, '/reseller_product_save')
form = ResellerProductForm(request.form)
if not form.validate_on_submit():
flash('Form validation error', 'danger')
log(log.WARNING, "Form validation error %s", form.errors)
return redirect(url_for('reseller.edit', id=form.reseller_id.data))
product = ResellerProduct.query.filter(
ResellerProduct.id == form.id.data).first()
if product:
product.reseller_id = form.reseller_id.data
product.product_id = form.product_id.data
product.months = form.months.data
product.init_price = form.init_price.data
product.ext_price = form.ext_price.data
else:
product = ResellerProduct.query.filter(
ResellerProduct.reseller_id == form.reseller_id.data).filter(
ResellerProduct.product_id == form.product_id.data).filter(
ResellerProduct.months == form.months.data).first()
if product:
flash('Reseller already have this product', 'danger')
return redirect(url_for('reseller.edit', id=form.reseller_id.data))
product = ResellerProduct(reseller_id=form.reseller_id.data,
product_id=form.product_id.data,
months=form.months.data,
init_price=form.init_price.data,
ext_price=form.ext_price.data)
product.save()
if ninja.configured:
product_key = ninja_product_name(product.product.name, product.months)
if form.id.data < 0:
ninja_product = ninja.add_product(product_key=product_key,
notes=product.reseller.name,
cost=product.init_price)
if ninja_product:
product.ninja_product_id = ninja_product.id
product.save()
else:
ninja_product = ninja.get_product(product.ninja_product_id)
if ninja_product:
ninja.update_product(ninja_product.id,
product_key=product_key,
notes=product.reseller.name,
cost=product.init_price)
return redirect(url_for('reseller.edit', id=form.reseller_id.data))
def save_update():
log(log.INFO, '%s /account_ext_save_update', request.method)
form = AccountExtensionForm(request.form)
if not form.validate_on_submit():
flash(VALIDATION_ERROR, 'danger')
log(log.WARNING, VALIDATION_ERROR, form.errors)
return redirect(url_for('account.edit', id=form.id.data))
extension = AccountExtension.query.filter(AccountExtension.id == form.id.data).first()
extension.reseller_id = form.reseller_id.data
extension.product_id = form.product_id.data
extension.months = form.months.data
extension.extension_date = form.extension_date.data
extension.end_date = extension.extension_date + relativedelta(months=extension.months)
account_id = extension.account_id
extension.save()
return redirect(url_for('account.edit', id=account_id))
def run_scrap():
try:
data_scrap()
except Exception as err:
# send email on error
import traceback
body = f"Error {str(err)}\n"
body += traceback.format_exc()
msg = Message(subject="Error",
body=body,
recipients=conf.MAIL_RECIPIENTS.split(";"))
log(log.INFO, "Mail sending...")
with open(f"{LOGGER_NAME}.log", "r") as att:
msg.attach(att.name, "text/plain", att.read())
mail.send(msg)
assert msg
