def __init__(self, dataname, datatype, params):
"""
dataname: sql filename
datatype: db engine or filetype in str
"""
self.source = dataname
self.type = datatype
logger.info(dataname + str(datatype))
if datatype in ("xls", "xlsx"):
self.data = pd.read_excel(
os.path.join(conf.get("PROJ_DIR"), "data", dataname))
elif datatype in ("csv", "txt"):
self.data = pd.read_csv(
os.path.join(conf.get("PROJ_DIR"), "data", dataname))
elif datatype in ("api", ):
api_cls = importlib.import_module("services." + dataname,
package="..")
api = api_cls.Api()
self.data = api.data
else:
str_sql = open(
os.path.join(conf.get("PROJ_DIR"), "sqls", dataname),
encoding="utf-8",
).read()
sql = set_sql_params(str_sql, params)
self.data = pd.read_sql(sql, datatype)
def ENGINE_MARIADB_dw(self):
return create_engine(
"mysql+pymysql://{}:{}@{}/{}".format(conf.get("MARIADB_USER"),
conf.get("MARIADB_PASS"),
conf.get("MARIADB_IP"),
conf.get("MARIADB_DB")),
connect_args={'charset': conf.get("MARIADB_CHARSET")})
def send(func):
"""
send msg to sns app called dingding
https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.2vzwCr&treeId=257&articleId=105735&docType=1
you should conif your token in sachima.yaml
"""
ERROR_GRP_TOKEN = conf.get("SNS_DINGDING_ERROR_GRP_TOKEN")
INFO_GRP_TOKEN = conf.get("SNS_DINGDING_INFO_GRP_TOKEN")
SENDING_STR = conf.get("SNS_DINGDING_SENDING_STR")
ERRSENT_STR = conf.get("SNS_DINGDING_ERRSENT_STR")
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
# before
time_str = str(datetime.datetime.now())
t = SENDING_STR.format(args[0]["handler"], time_str)
sns.send_dingding(t, t, ERROR_GRP_TOKEN)
value = func(*args, **kwargs)
# after
return value
except Exception as ex:
title = ERRSENT_STR.format(args[0]["handler"],
str(datetime.datetime.now()))
data = str(ex)
sns.send_dingding(title, title + data, ERROR_GRP_TOKEN)
sns.send_dingding(title, title + data, INFO_GRP_TOKEN)
return wrapper
def ENGINE_IMPALA_DW(self):
_impala_conn = imp_connect(
host=conf.get("IMPALA_IP"),
port=conf.get("IMPALA_PORT"),
database=conf.get("IMPALA_DB"),
timeout=conf.get("IMPALA_TIMEOUT"),
# use_ssl=True,
# ca_cert='some_pem',
# user='******',
# password='******',
auth_mechanism=conf.get("IMPALA_AUTH_MECHANISM"),
# kerberos_service_name='hive'
)
return create_engine('impala://', creator=_impala_conn, echo=False)
def CONN_MYSQL_d(self):
return my_connect(host=conf.get("MYSQL_D_IP"),
user=conf.get("MYSQL_D_USER"),
password=conf.get("MYSQL_D_PASS"),
port=conf.get("MYSQL_D_PORT"),
database=conf.get("MYSQL_D_DB"),
charset=conf.get("MYSQL_D_CHARSET"),
cursorclass=cursors.SSDictCursor)
def CONN_MYSQL_h(self):
return my_connect(host=conf.get("MYSQL_H_IP"),
user=conf.get("MYSQL_H_USER"),
password=conf.get("MYSQL_H_PASS"),
port=conf.get("MYSQL_H_PORT"),
database=conf.get("MYSQL_H_DB"),
charset=conf.get("mysql_h_charset"),
cursorclass=cursors.SSDictCursor)
def to_superset(self, name, type_, param):
addr = conf.get("SUPERSET_WEBSERVER_ADDRESS")
port = conf.get("SUPERSET_WEBSERVER_PORT")
user = conf.get("SUPERSET_USERNAME")
pwd = conf.get("SUPERSET_PASSWORD")
bp_post = conf.get("SUPERSET_API_TABLE_BP")
if addr and port:
url = ":".join([addr.rstrip("/"), str(port)])
with requests.session() as s:
# 登陆
r = s.get(url + "/login/")
login_data = dict(
username=user,
password=pwd,
csrf_token=self.get_csrf_token(r.text),
)
r = s.post(url + "/login/", data=login_data)
# 调用接口
if r.url.endswith("welcome"):
r = s.post(
url + bp_post,
headers={
"Content-Type": "application/json; charset=utf-8",
"X-CSRFToken": self.get_csrf_token(r.text),
},
json={
"slice_name": name,
"api": type_,
"params": param,
},
)
print(r.text)
print("publish service to superset")
else:
pass
import smtplib
from os.path import basename
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate, formataddr, parseaddr
from email.header import Header
from sachima import conf
mail_host = conf.get("MAIL_HOST")
MAIL_ADD = conf.get("MAIL_ADD")
MAIL_USER = conf.get("MAIL_USER")
MAIL_PASS = conf.get("MAIL_PASS")
MAIL_SENDER = conf.get("MAIL_SENDER")
def _format_addr(s):
name, addr = parseaddr(s)
return formataddr((Header(name, 'utf-8').encode(), addr))
def send_mail(send_to,
cc_to,
subject,
text,
files=None,
server=mail_host,
ishtml=False):
assert isinstance(send_to, list)
msg = MIMEMultipart()
from nameko.standalone.rpc import ClusterRpcProxy
from sanic import Sanic
from sanic.response import json
from sachima import conf
app = Sanic()
CONFIG = conf.get("BROKER")
def sachima_rpc_reports(req):
with ClusterRpcProxy(CONFIG) as rpc:
# print(req)
res = rpc.data.get_report(req.json)
return res
@app.route("/reports", methods=["POST"])
async def sachima(request):
# print(request)
return json(sachima_rpc_reports(request))
"qualname": "sachima.error",
},
},
handlers={
"console": {
"class": "logging.handlers.RotatingFileHandler",
"formatter": "generic",
"filename": "logs/sachima.log", # noqa
"maxBytes": 1024000,
"backupCount": 10,
# "stream": sys.stdout,
},
"error_console": {
"class": "logging.StreamHandler",
"formatter": "generic",
"stream": sys.stderr,
},
},
formatters={
"generic": {
"format": "%(asctime)s [%(process)d] [%(levelname)s] %(message)s",
"datefmt": "[%Y-%m-%d %H:%M:%S %z]",
"class": "logging.Formatter",
}
},
)
logging.config.dictConfig(LOGGING_CONFIG_DEFAULTS)
logger = logging.getLogger("sachima.root")
logger.setLevel(conf.get("LOG_LEVEL"))
def CONN_IMPALA_DW(self):
return imp_connect(host=conf.get("IMPALA_IP"),
port=conf.get("IMPALA_PORT"),
database=conf.get("IMPALA_DB"),
auth_mechanism=conf.get("IMPALA_AUTH_MECHANISM"))
def ENGINE_SUPERSET(self):
return create_engine('sqlite:///{}'.format(conf.get("DB_SUPERSET")),
echo=True)