def read(self, msg: MSG):
# logging.debug("FILE::{0}::READ msg:{1}".format(self.conn, msg))
# -------------------------------------------------
rm_idx = []
for i in range(0, msg.args_count, 1):
arguments = msg.read_args(i)
if self.url_is_dir:
path = os.path.join(self.conn.path, arguments["path"])
else:
path = self.conn.path
suffix = os.path.splitext(path)[-1]
# -------------------------------
if os.path.isfile(path):
if suffix in [".xls", ".xlsx"]:
if self.url_is_dir:
datum = pd.read_excel(path)
else:
datum = pd.read_excel(path,
sheet_name=arguments["path"])
elif suffix in [".npy"]:
datum = np.load(path)
else:
with open(path, "rb") as file:
datum = file.read()
msg.add_datum(datum, arguments["path"])
logging.info("FILE::{0}::READ successfully.".format(path))
else:
logging.warning("FILE::{0}::READ failed.".format(path))
rm_idx = [i] + rm_idx
# -------------------------------
if CONFIG.IS_DATA_READ_START:
for i in rm_idx:
msg.remove_args(i)
logging.info("FILE::{0}::READ successfully.".format(self.conn))
return msg
def demo_numpy():
arr = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
msg = MSG()
msg.add_datum(arr)
message = str(msg)
new_msg = MSG(message)
print(msg)
print(new_msg)
print(new_msg.read_datum(0)["stream"])
def demo_excel():
path = "C:/Users/geyua/PycharmProjects/Babelor/data/dir1/20190505.xlsx"
df = pd.read_excel(path)
msg = MSG()
msg.add_datum(df, path)
message = str(msg)
new_msg = MSG(message)
print(msg)
print(new_msg)
def read(self, msg: MSG):
# logging.debug("FTP::{0}::READ msg:{1}".format(self.conn, msg))
ftp = self.open()
# -------------------------------------------------
rm_idx = []
for i in range(0, msg.args_count, 1):
argument = msg.read_args(i)
if self.url_is_dir:
path = os.path.join(self.conn.path, argument["path"])
else:
path = self.conn.path
suffix = os.path.splitext(path)[-1]
# ----------------------------
stream = bytes()
ftp.retrbinary('RETR ' + path, stream, CONFIG.FTP_BUFFER)
temp_path = "temp/temp" + suffix
mkdir(os.path.split(temp_path)[0])
# -------------------------------
if suffix in [".xls", ".xlsx"]:
with open(temp_path, "wb") as temp_file:
temp_file.write(stream)
if self.url_is_dir:
stream = pd.read_excel(temp_path)
else:
stream = pd.read_excel(temp_path,
sheet_name=argument["path"])
logging.info(
"FTP::EXCEL::{0}::READ successfully.".format(path))
elif suffix in [".npy"]:
with open(temp_path, "wb") as temp_file:
temp_file.write(stream)
stream = np.load(temp_path)
logging.info(
"FTP::NUMPY::{0}::READ successfully.".format(path))
else:
logging.info("FTP::{0}::READ successfully.".format(path))
os.remove(temp_path)
os.removedirs(os.path.split(temp_path)[0])
del temp_path
# -------------------------------
msg.add_datum(datum=stream, path=argument["path"])
rm_idx = [i] + rm_idx
# -------------------------------------------------
if CONFIG.IS_DATA_READ_START:
for i in rm_idx:
msg.remove_args(i)
logging.info("FTP::{0}::READ successfully.".format(self.conn))
return msg
def read(self, msg: MSG):
# logging.debug("SQL::{0}::READ msg:{1}".format(self.conn, msg))
# ----------------------------------
rm_idx = []
for i in range(0, msg.args_count, 1):
argument = msg.read_args(i)
df = pd.read_sql(sql=argument["stream"], con=self.engine)
df = df.rename(str.upper, axis='columns')
msg.add_datum(datum=df, path=argument["path"])
rm_idx = [i] + rm_idx
# ----------------------------------
if CONFIG.IS_DATA_READ_START:
for i in rm_idx:
msg.remove_args(i)
logging.info("SQL::{0}::READ successfully.".format(self.conn))
return msg
def demo_msg_mysql2ftp():
msg = MSG()
msg.origination = URL().init("mysql").check
msg.destination = URL().init("ftp").check
msg.treatment = URL().init("tcp")
case = CASE()
case.origination = msg.origination
case.destination = msg.destination
msg.activity = "init"
msg.case = case
msg.add_datum("This is a test string", path=URL().init("file"))
msg.add_datum("中文UTF-8编码测试", path=URL().init("file"))
path = "..\README\Babelor-设计.png"
with open(path, "rb") as f:
bytes_f = f.read()
url = URL().init("file")
url.path = path
msg.add_datum(bytes_f, url)
msg_string = str(msg)
new_msg = MSG(msg_string)
new_bytes_f = new_msg.read_datum(3)["stream"]
new_path = "..\README\Babelor-设计-new.png"
with open(new_path, "wb") as f:
f.write(new_bytes_f)
receiver_msg.origination = edge_node_url["inner"]
receiver_msg.destination = destination_url
# logging.warning("RECEIVER::INIT::{0} send:{1}".format(receiver_url["inner"], receiver_msg))
recv_init = MQ(receiver_url["outer"])
recv_init.push(receiver_msg)
def sender_init():
# -————————————------------------------ MESSAGE -----
case = CASE("{0}#{1}".format(origination_url, destination_url))
sender_msg = MSG()
sender_msg.case = case
# -————————————------------------------ SENDER ------
sender_msg.origination = origination_url
sender_msg.destination = edge_node_url["outer"]
sender_msg.add_datum(None, "20190505.xlsx")
sender_msg.add_datum(None, "20190506.xlsx")
sender_msg.add_datum(None, "20190507.xlsx")
send_init = MQ(sender_url["outer"])
send_init.push(sender_msg)
def main():
# -————————————------------------------ PROCESS ------
temple = {
# "treater": Process(target=treater, args=(treater_url["inner"],)),
# "encrypter": Process(target=encrypter, args=(encrypter_url["inner"],)),
"receiver": Process(target=receiver, args=(receiver_url["inner"],)),
"receiver_init": Process(target=receiver_init),
"sender": Process(target=sender, args=(sender_url["inner"],)),
"sender_init": Process(target=sender_init),