def __init__(self, conn: URL):
# conn = "oracle://*****:*****@hostname:1521/service"
# conn = "mysql://*****:*****@hostname:3306/service"
if isinstance(conn, str):
self.conn = URL(conn)
else:
self.conn = conn
self.conn = self.__dict__["conn"].check
self.engine = create_engine(self.conn.to_string())
def try_push(url: str):
msg = MSG()
msg.origination = URL("tcp://127.0.0.1:10001")
msg.destination = URL("tcp://127.0.0.1:10001")
mq = MQ(url)
for i in range(0, 10000, 1):
# print("push msg:", msg)
msg.activity = str(i)
# print("push msg:", i, msg)
mq.push(msg)
def demo_msg_mysql2ftp():
msg = MSG()
msg.origination = URL().init("mysql").check
msg.destination = URL().init("ftp").check
msg.treatment = URL().init("tcp")
case = CASE()
case.origination = msg.origination
case.destination = msg.destination
msg.activity = "init"
msg.case = case
msg.add_datum("This is a test string", path=URL().init("file"))
msg.add_datum("中文UTF-8编码测试", path=URL().init("file"))
path = "..\README\Babelor-设计.png"
with open(path, "rb") as f:
bytes_f = f.read()
url = URL().init("file")
url.path = path
msg.add_datum(bytes_f, url)
msg_string = str(msg)
new_msg = MSG(msg_string)
new_bytes_f = new_msg.read_datum(3)["stream"]
new_path = "..\README\Babelor-设计-new.png"
with open(new_path, "wb") as f:
f.write(new_bytes_f)
def demo_tcp_url():
url = URL().init("tcp")
url = url.check
print("\nURL:", url)
print("服务协议:", url.scheme)
print("服务地址", url.hostname)
print("服务端口", url.port)
def __init__(self, conn: URL):
if isinstance(conn, str):
self.conn = URL(conn)
else:
self.conn = conn
if os.path.splitext(self.conn.path)[-1] in [""]:
self.url_is_dir = True
else:
self.url_is_dir = False
def demo_oracle_url():
url = URL().init("oracle")
url = url.check
print("\nURL:", url)
print("服务协议:", url.scheme)
print("服务用户:", url.username)
print("服务密码:", url.password)
print("服务地址", url.hostname)
print("服务端口", url.port)
print("数据服务", url.path)
def demo_ftp_url():
url = URL().init("ftp")
url = url.check
print("\nURL:", url)
print("服务协议:", url.scheme)
print("服务用户:", url.username)
print("服务密码:", url.password)
print("服务地址", url.hostname)
print("服务端口", url.port)
print("服务路径", url.path)
print("服务模式", url.fragment.query["model"])
def __init__(self, conn: (URL, str)):
if isinstance(conn, str):
self.conn = URL(conn)
else:
self.conn = conn
# Check
if self.conn.scheme not in ["tcp", "pgm", "inproc"]:
raise ValueError("Invalid scheme{0}.".format(self.conn.scheme))
self.pipe_in = Pipe() # PIPE IN
self.pipe_out = Pipe() # PIPE OUT
self.queue_ctrl = Queue(CONFIG.MQ_MAX_DEPTH) # QUEUE CTRL
self.active = False # 激活状态
self.initialed = None # 初始化模式
self.process = None # 队列进程
class SQL:
def __init__(self, conn: URL):
# conn = "oracle://*****:*****@hostname:1521/service"
# conn = "mysql://*****:*****@hostname:3306/service"
if isinstance(conn, str):
self.conn = URL(conn)
else:
self.conn = conn
self.conn = self.__dict__["conn"].check
self.engine = create_engine(self.conn.to_string())
def read(self, msg: MSG):
# logging.debug("SQL::{0}::READ msg:{1}".format(self.conn, msg))
# ----------------------------------
rm_idx = []
for i in range(0, msg.args_count, 1):
argument = msg.read_args(i)
df = pd.read_sql(sql=argument["stream"], con=self.engine)
df = df.rename(str.upper, axis='columns')
msg.add_datum(datum=df, path=argument["path"])
rm_idx = [i] + rm_idx
# ----------------------------------
if CONFIG.IS_DATA_READ_START:
for i in rm_idx:
msg.remove_args(i)
logging.info("SQL::{0}::READ successfully.".format(self.conn))
return msg
def write(self, msg: MSG):
# logging.debug("SQL::{0} write:{1}".format(self.conn, msg))
# ----------------------------------
rm_idx = []
for i in range(0, msg.dt_count, 1):
rt = msg.read_datum(i)
df = rt["stream"]
path = os.path.splitext(rt["path"])[0]
if isinstance(df, pd.DataFrame):
df.to_sql(path,
con=self.engine,
if_exists='replace',
index=False,
index_label=False)
logging.info("SQL::{0}::WRITE successfully.".format(self.conn))
else:
logging.warning("SQL::{0}::WRITE failed.".format(self.conn))
rm_idx = [i] + rm_idx
# ----------------------------------
if CONFIG.IS_DATA_WRITE_END:
for i in rm_idx:
msg.remove_datum(i)
def __init__(self, conn: URL):
# "tomail://<receiver.mail.username>@<receive.mail.hostname>/<receiver>"
# "smtp://<sender.username>:<sender.password>@<sender.hostname>:<port>"
# "tomail://<sender.mail.username>@<sender.mail.hostname>/<sender>"
if isinstance(conn, str):
self.conn = URL(conn)
else:
self.conn = conn
self.conn = self.__dict__["conn"].check
self.mime = MIMEMultipart()
self.me = None
self.to = None
self.subject = None
self.content = None
def demo_tomail_url():
url = URL().init("tomail+smtp")
url = url.check
print("URL:", url)
print("收件人邮箱:", url.netloc)
print("收件人用户名:", url.username)
print("收件人名:", url.path)
print("发件人邮箱：", url.fragment.fragment.netloc)
print("发件人用户名:", url.fragment.username)
print("发件人名:", url.fragment.fragment.path)
print("服务协议:", url.fragment.scheme)
print("服务用户:", url.fragment.username)
print("服务密码:", url.fragment.password)
print("服务地址", url.fragment.hostname)
print("服务端口", url.fragment.port)
def __init__(self, conn: URL):
if isinstance(conn, str):
self.conn = URL(conn)
else:
self.conn = conn
self.conn = self.__dict__["conn"].check
def try_subscribe():
mq = MQ(URL("tcp://127.0.0.1:10001"))
msg = mq.subscribe()
print("subscribe msg:", msg)
def try_publish():
msg = MSG()
msg.origination = URL("tcp://127.0.0.1:10001")
mq = MQ(URL("tcp://*:10001"))
print("publish msg:", msg)
mq.publish(msg)
def try_reply():
mq = MQ(URL("tcp://*:10001"))
mq.reply(try_reply_func)
def try_reply_func(msg: MSG):
msg.destination = URL().init("oracle")
return msg
def try_request():
msg = MSG()
msg.origination = URL("tcp://127.0.0.1:10001")
mq = MQ(URL("tcp://127.0.0.1:10001"))
msg = mq.request(msg)
"receiver": Process(target=receiver, args=(receiver_url["inner"],)),
"receiver_init": Process(target=receiver_init),
"sender": Process(target=sender, args=(sender_url["inner"],)),
"sender_init": Process(target=sender_init),
}
# for obj in temple.items():
# key, value = obj
# value.start()
temple["receiver"].start()
temple["sender"].start()
temple["receiver_init"].start()
temple["sender_init"].start()
sender_url = {
"inner": URL("tcp://*:20001"),
"outer": URL("tcp://127.0.0.1:20001"),
}
treater_url = {
"inner": URL("tcp://*:20002"),
"outer": URL("tcp://127.0.0.1:20002"),
}
encrypter_url = {
"inner": URL("tcp://*:20003"),
"outer": URL("tcp://127.0.0.1:20003"),
}
receiver_url = {
"inner": URL("tcp://*:20004"),
"outer": URL("tcp://127.0.0.1:20004"),
}
edge_node_url = {